{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 导入数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('RentListingInquries_FE_train.csv')\n",
    "train = train.head(2000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAELCAYAAADOeWEXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAGEhJREFUeJzt3Xu0JWV95vHvAwgRbw3SOqQvadSO\nSrxBjoqSMQheQB1AlkRYjrTKSoc1qDjGBIwOTMRMYrxFYiTTM6Aww3D1QmdEkRCQEQVpUO4aOkig\nhUAjF1GDLPA3f+w6YdOcPqfqnLPP3qfP97PWXrvqrbd2/fAsfKh6q95KVSFJUltbDbsASdL8YnBI\nkjoxOCRJnRgckqRODA5JUicGhySpE4NDktSJwSFJ6mRgwZHk5CR3Jblugm0fSFJJdmrWk+SEJOuT\nXJNk976+q5Lc1HxWDapeSVI72wzwt78AfBY4tb8xyTLgtcCtfc37ASubz8uBE4GXJ9kROA4YAwq4\nMsnaqrp3sgPvtNNOtWLFitn5p5CkBeLKK6+8u6oWT9VvYMFRVZckWTHBpk8Dfwyc29d2AHBq9eY/\nuSzJoiQ7A3sBF1TVPQBJLgD2BU6f7NgrVqxg3bp1M/5nkKSFJMk/t+k3p2McSfYHflxVV2+yaQlw\nW9/6hqZtc+2SpCEZ5KWqx0iyPfAh4HUTbZ6grSZpn+j3VwOrAZYvXz7NKiVJU5nLM45nA7sAVye5\nBVgKXJXk39E7k1jW13cpcPsk7Y9TVWuqaqyqxhYvnvISnSRpmuYsOKrq2qp6RlWtqKoV9EJh96r6\nF2AtcFhzd9UewP1VdQdwPvC6JDsk2YHe2cr5c1WzJOnxBnk77unAd4DnJtmQ5PBJup8H3AysB/4H\n8J8AmkHx44Erms9HxgfKJUnDkS3xRU5jY2PlXVWS1E2SK6tqbKp+PjkuSerE4JAkdWJwSJI6mbPn\nOCRpU3v+9Z7DLmGLd+l7Lp313/SMQ5LUicEhSerE4JAkdWJwSJI6MTgkSZ0YHJKkTgwOSVInBock\nqRODQ5LUicEhSerE4JAkdWJwSJI6MTgkSZ0YHJKkTgwOSVInBockqRODQ5LUicEhSepkYMGR5OQk\ndyW5rq/t40l+kOSaJF9Osqhv2weTrE/ywySv72vft2lbn+SYQdUrSWpnkGccXwD23aTtAuAFVfUi\n4B+BDwIk2RU4BPitZp/PJdk6ydbA3wD7AbsChzZ9JUlDMrDgqKpLgHs2aftGVT3crF4GLG2WDwDO\nqKpfVtWPgPXAy5rP+qq6uaoeAs5o+kqShmSYYxzvAr7WLC8BbuvbtqFp21y7JGlIhhIcST4EPAyc\nNt40QbeapH2i31ydZF2SdRs3bpydQiVJjzPnwZFkFfAm4G1VNR4CG4Blfd2WArdP0v44VbWmqsaq\namzx4sWzX7gkCZjj4EiyL3A0sH9V/aJv01rgkCTbJdkFWAl8F7gCWJlklyTb0htAXzuXNUuSHmub\nQf1wktOBvYCdkmwAjqN3F9V2wAVJAC6rqiOq6vokZwE30LuEdWRVPdL8zruB84GtgZOr6vpB1SxJ\nmtrAgqOqDp2g+aRJ+v8Z8GcTtJ8HnDeLpUmSZsAnxyVJnRgckqRODA5JUicGhySpE4NDktSJwSFJ\n6sTgkCR1YnBIkjoxOCRJnRgckqRODA5JUicGhySpE4NDktTJlMGR5ElJtmqWfzPJ/kmeMPjSJEmj\nqM0ZxyXAryVZAlwIvBP4wiCLkiSNrjbBkeZtfQcBf11VbwZ2HWxZkqRR1So4krwCeBvw1aZtYC+A\nkiSNtjbB8T56r3z9cvOK12cBFw22LEnSqJryzKGqvgl8M8mTmvWbgfcOujBJ0mhqc1fVK5LcANzY\nrL84yecGXpkkaSS1uVT1V8DrgZ8AVNXVwKsGWZQkaXS1egCwqm7bpOmRAdQiSZoH2gTHbUleCVSS\nbZN8gOay1WSSnJzkriTX9bXtmOSCJDc13zs07UlyQpL1Sa5JsnvfPqua/jclWTWNf0ZJ0ixqExxH\nAEcCS4ANwEua9al8Adh3k7ZjgAuraiW9hwmPadr3A1Y2n9XAidALGuA44OXAy4DjxsNGkjQcbe6q\nupveMxydVNUlSVZs0nwAsFezfApwMXB0035qVRVwWZJFSXZu+l5QVfcAJLmAXhid3rUeSdLsaHNX\n1SlJFvWt75Dk5Gke75lVdQdA8/2Mpn0J0D+OsqFp21z7RHWuTrIuybqNGzdOszxJ0lTaXKp6UVXd\nN75SVfcCu81yHZmgrSZpf3xj1ZqqGquqscWLF89qcZKkR7UJjq36xxWacYfpTjlyZ3MJiub7rqZ9\nA7Csr99S4PZJ2iVJQ9ImOD4JfDvJ8UmOB74N/OU0j7cWGL8zahVwbl/7Yc3dVXsA9zeXss4HXtdc\nHtsBeF3TJkkakjaD46cmuRJ4Nb1LRwdV1Q1T7ZfkdHqD2zsl2UDv7qi/AM5KcjhwK3Bw0/084A3A\neuAX9KZup6ruacLqiqbfR8YHyiVJw9H2ktMPgHvH+ydZXlW3TrZDVR26mU37TNC32MwtvlV1MjDd\nwXhJ0iybMjiSvIfe2cKd9J4YD70B6hcNtjRJ0ihqc8ZxFPDcqvrJoIuRJI2+VlOOAPcPuhBJ0vzQ\n5ozjZuDiJF8FfjneWFWfGlhVkqSR1SY4bm0+2zYfSdIC1uZ23D8FSPKkqvr54EuSJI0y3wAoSerE\nNwBKkjrxDYCSpE7aDI4/5g2AwHtp8QZASdKWaZBvAJQkbYEmPeNIsjXw9qrq/AZASdKWadIzjqp6\nhN5rXSVJAtqNcVya5LPAmcC/PcdRVVcNrCpJ0shqExyvbL4/0tdWwN6zX44kadRNNcaxFXBiVZ01\nR/VIkkbcVGMcvwLePUe1SJLmgTa3416Q5ANJliXZcfwz8MokSSOpzRjHu5rv/mc3CnjW7JcjSRp1\nbWbH3WUuCpEkzQ9t3jl+2ETtVXXq7JcjSRp1bcY4Xtr3+ffAfwX2n8lBk/znJNcnuS7J6Ul+Lcku\nSS5PclOSM5t5sUiyXbO+vtm+YibHliTNzJTBUVXv6fv8PrAbM3gTYJIl9CZKHKuqFwBbA4cAHwM+\nXVUrgXuBw5tdDgfurarnAJ9u+kmShqTVtOqb+AWwcobH3QZ4YpJtgO2BO+g9UHhOs/0U4MBm+YBm\nnWb7Pkkyw+NLkqapzRjH39G7iwp6QbMrMO0HAqvqx0k+Qe895v8KfAO4Erivqh5uum2gNxsvzfdt\nzb4PJ7kfeDpw93RrkCRNX5vbcT/Rt/ww8M9VtWG6B0yyA72ziF2A+4Czgf0m6DoeVhOdXdSmDUlW\nA6sBli9fPt3yJElTaBMctwJ3VNWDAEmemGRFVd0yzWO+BvhRVW1sfu9L9ObDWpRkm+asYylwe9N/\nA7AM2NBc2noacM+mP1pVa4A1AGNjY48LFknS7GgzxnE28Ku+9Ueatum6FdgjyfbNWMU+wA3ARcBb\nmj6rgHOb5bXNOs32f6gqg0GShqRNcGxTVQ+NrzTL076rqqoupzfIfRVwbVPDGuBo4P1J1tMbwzip\n2eUk4OlN+/uBY6Z7bEnSzLW5VLUxyf5VtRYgyQHMcGC6qo4Djtuk+WbgZRP0fRA4eCbHkyTNnjbB\ncQRwWvMyJ+iNOUz4NLkkacvXZq6qf6I3JvFkIFX1wODLkiSNqinHOJL8tySLqupnVfVAkh2SfHQu\nipMkjZ42g+P7VdV94ytVdS/whsGVJEkaZW2CY+sk242vJHkisN0k/SVJW7A2g+P/G7gwyefpPbH9\nLh6dO0qStMC0GRz/yyTX0HviG+D4qjp/sGVJkkZVmzMOgO8BT6B3xvG9wZUjSRp1be6q+j3gu/Sm\n+/g94PIkb5l8L0nSlqrNGceHgJdW1V0ASRYDf8+j786QJC0gbe6q2mo8NBo/abmfJGkL1OaM4+tJ\nzgdOb9bfCpw3uJIkSaOszV1Vf5TkIOB36L1UaU1VfXnglUmSRlKru6qq6kvAlwZciyRpHnCsQpLU\nicEhSepks8GR5MLm+2NzV44kadRNNsaxc5LfBfZPcga9gfF/U1VXDbQySdJImiw4jqX3fu+lwKc2\n2VbA3oMqSpI0ujYbHFV1DnBOkv9SVcfPYU2SpBHW5jmO45PsD7yqabq4qv7vYMuSJI2qNpMc/jlw\nFHBD8zmqaZMkLUBtbsd9I/Daqjq5qk4G9m3api3JoiTnJPlBkhuTvCLJjkkuSHJT871D0zdJTkiy\nPsk1SXafybElSTPT9jmORX3LT5uF434G+HpVPQ94MXAjvYH4C6tqJXBhsw6wH7Cy+awGTpyF40uS\npqnNlCN/DnwvyUX0bsl9FfDB6R4wyVOb33gHQFU9BDyU5ABgr6bbKcDFwNHAAcCpVVXAZc3Zys5V\ndcd0a5AkTV+bwfHTk1wMvJRecBxdVf8yg2M+C9gIfD7Ji4Er6Y2hPHM8DKrqjiTPaPovAW7r239D\n02ZwSNIQtLpUVVV3VNXaqjp3hqEBvbDaHTixqnYDfs6jl6Umkgna6nGdktVJ1iVZt3HjxhmWKEna\nnGHMVbUB2FBVlzfr59ALkjuT7AzQfN/V139Z3/5Lgds3/dGqWlNVY1U1tnjx4oEVL0kL3ZwHR3PG\ncluS5zZN+9C7zXctsKppWwWc2yyvBQ5r7q7aA7jf8Q1JGp5JxziSbAVcU1UvmOXjvgc4Lcm2wM3A\nO+mF2FlJDgduBQ5u+p4HvAFYD/yi6StJGpJJg6OqfpXk6iTLq+rW2TpoVX0fGJtg0z4T9C3gyNk6\ntiRpZtrcjrszcH2S79IbyAagqvYfWFWSpJHVJjj+dOBVSJLmjTbPcXwzyW8AK6vq75NsD2w9+NIk\nSaOozSSHv0/vltn/3jQtAb4yyKIkSaOrze24RwJ7Aj8FqKqbgGdMuockaYvVJjh+2cwnBUCSbZjg\nyW1J0sLQJji+meRPgCcmeS1wNvB3gy1LkjSq2gTHMfQmJbwW+AN6D+R9eJBFSZJGV5u7qn6V5BTg\ncnqXqH7YPJQnSVqApgyOJG8E/hb4J3oz1e6S5A+q6muDLk6SNHraPAD4SeDVVbUeIMmzga8CBock\nLUBtxjjuGg+Nxs08OuW5JGmB2ewZR5KDmsXrk5wHnEVvjONg4Io5qE2SNIImu1T1H/qW7wR+t1ne\nCOwwsIqkDm79yAuHXcIWb/mx1w67BI2YzQZHVfneC0nS47S5q2oXei9eWtHf32nVJWlhanNX1VeA\nk+g9Lf6rwZYjSRp1bYLjwao6YeCVSJLmhTbB8ZkkxwHfAH453lhVVw2sKknSyGoTHC8E3g7szaOX\nqqpZlyQtMG2C483As/qnVpckLVxtnhy/Glg06EIkSfNDmzOOZwI/SHIFjx3jmNHtuEm2BtYBP66q\nNzW3/Z4B7AhcBby9qh5Ksh1wKvDbwE+At1bVLTM5tiRp+toEx3EDOvZRwI3AU5v1jwGfrqozkvwt\ncDhwYvN9b1U9J8khTb+3DqgmSdIUprxUVVXfnOgzk4MmWQq8EfifzXroDbaf03Q5BTiwWT6gWafZ\nvk/TX5I0BFMGR5IHkvy0+TyY5JEkP53hcf8K+GMevUvr6cB9VfVws74BWNIsLwFuA2i239/0lyQN\nQZs3AD6lfz3JgcDLpnvAJG+iN1X7lUn2Gm+e6NAttvX/7mpgNcDy5cunW54kaQpt7qp6jKr6CjN7\nhmNPYP8kt9AbDN+b3hnIoiTjQbYUuL1Z3gAsA2i2Pw24Z4K61lTVWFWNLV68eAblSZIm02aSw4P6\nVrcCxpjgv/jbqqoPAh9sfnsv4ANV9bYkZwNvoRcmq4Bzm13WNuvfabb/g+88l6ThaXNXVf97OR4G\nbqE3YD3bjgbOSPJR4Hv0Jlak+f5fSdbTO9M4ZADHliS11GaMY2Dv5aiqi4GLm+WbmWDspKoepPfW\nQUnSCJjs1bHHTrJfVdXxA6hHkjTiJjvj+PkEbU+i90De0wGDQ5IWoMleHfvJ8eUkT6H3pPc76Q1e\nf3Jz+0mStmyTjnEk2RF4P/A2ek9v715V985FYZKk0TTZGMfHgYOANcALq+pnc1aVJGlkTfYA4B8C\nvw58GLi9b9qRB2ZhyhFJ0jw12RhH56fKJUlbPsNBktSJwSFJ6sTgkCR1YnBIkjoxOCRJnRgckqRO\nDA5JUicGhySpE4NDktSJwSFJ6sTgkCR1YnBIkjoxOCRJnRgckqRO5jw4kixLclGSG5Ncn+Sopn3H\nJBckuan53qFpT5ITkqxPck2S3ee6ZknSo4ZxxvEw8IdV9XxgD+DIJLsCxwAXVtVK4MJmHWA/YGXz\nWQ2cOPclS5LGzXlwVNUdVXVVs/wAcCOwBDiA3nvNab4PbJYPAE6tnsuARUl2nuOyJUmNoY5xJFkB\n7AZcDjyzqu6AXrgAz2i6LQFu69ttQ9MmSRqCoQVHkicDXwTeV1WTvcM8E7TVBL+3Osm6JOs2btw4\nW2VKkjYxlOBI8gR6oXFaVX2pab5z/BJU831X074BWNa3+1Lg9k1/s6rWVNVYVY0tXrx4cMVL0gI3\njLuqApwE3FhVn+rbtBZY1SyvAs7taz+subtqD+D+8UtakqS5t80Qjrkn8Hbg2iTfb9r+BPgL4Kwk\nhwO3Agc3284D3gCsB34BvHNuy5Uk9Zvz4KiqbzHxuAXAPhP0L+DIgRYlSWptGGccI+e3/+jUYZew\nxbvy44cNuwRJs8QpRyRJnRgckqRODA5JUicGhySpE4NDktSJwSFJ6sTgkCR1YnBIkjoxOCRJnRgc\nkqRODA5JUicGhySpE4NDktSJwSFJ6sTgkCR1YnBIkjoxOCRJnRgckqRODA5JUicGhySpE4NDktTJ\nvAmOJPsm+WGS9UmOGXY9krRQzYvgSLI18DfAfsCuwKFJdh1uVZK0MM2L4ABeBqyvqpur6iHgDOCA\nIdckSQvSfAmOJcBtfesbmjZJ0hzbZtgFtJQJ2uoxHZLVwOpm9WdJfjjwqoZnJ+DuYRfRRT6xatgl\njJL59fc7bqJ//Ras+fW3A/LeTn+/32jTab4ExwZgWd/6UuD2/g5VtQZYM5dFDUuSdVU1Nuw6ND3+\n/eYv/3Y98+VS1RXAyiS7JNkWOARYO+SaJGlBmhdnHFX1cJJ3A+cDWwMnV9X1Qy5LkhakeREcAFV1\nHnDesOsYEQviktwWzL/f/OXfDkhVTd1LkqTGfBnjkCSNCINjnnHqlfkryclJ7kpy3bBrUTdJliW5\nKMmNSa5PctSwaxomL1XNI83UK/8IvJbeLcpXAIdW1Q1DLUytJHkV8DPg1Kp6wbDrUXtJdgZ2rqqr\nkjwFuBI4cKH+u+cZx/zi1CvzWFVdAtwz7DrUXVXdUVVXNcsPADeygGevMDjmF6dekYYsyQpgN+Dy\n4VYyPAbH/DLl1CuSBifJk4EvAu+rqp8Ou55hMTjmlymnXpE0GEmeQC80TquqLw27nmEyOOYXp16R\nhiBJgJOAG6vqU8OuZ9gMjnmkqh4GxqdeuRE4y6lX5o8kpwPfAZ6bZEOSw4ddk1rbE3g7sHeS7zef\nNwy7qGHxdlxJUieecUiSOjE4JEmdGBySpE4MDklSJwaHJKkTg0OS1InBoQUlybdb9Hlfku0HXMdL\npnoOIMk7knx2lo8767+phcfg0IJSVa9s0e19QKfgaKa87+IlwIJ9gEzzm8GhBSXJz5rvvZJcnOSc\nJD9Iclp63gv8OnBRkouavq9L8p0kVyU5u5nojiS3JDk2ybeAg5M8O8nXk1yZ5P8leV7T7+Ak1yW5\nOsklzXQxHwHe2jyB/NYWdS9O8sUkVzSfPZNs1dSwqK/f+iTPnKj/rP+PqQVrm2EXIA3RbsBv0Zso\n8lJgz6o6Icn7gVdX1d1JdgI+DLymqn6e5Gjg/fT+jx/gwar6HYAkFwJHVNVNSV4OfA7YGzgWeH1V\n/TjJoqp6KMmxwFhVvbtlrZ8BPl1V30qyHDi/qp6f5FzgzcDnm2PeUlV3Jvk/m/YHnj/D/70kwODQ\nwvbdqtoAkOT7wArgW5v02QPYFbi0N88d29Kbb2rcmc3+TwZeCZzd9APYrvm+FPhCkrOA6c6q+hpg\n177ffmrzJroz6QXT5+lNennmFP2lGTM4tJD9sm/5ESb+9yHABVV16GZ+4+fN91bAfVX1kk07VNUR\nzdnAG4HvJ3lcnxa2Al5RVf/6mOKS7wDPSbIYOBD46BT9p3Fo6bEc45Ae7wFg/L/OLwP2TPIcgCTb\nJ/nNTXdoXurzoyQHN/2S5MXN8rOr6vKqOha4m947VfqP0cY36M2MTPObL2mOW8CXgU/Rm/L7J5P1\nl2aDwSE93hrga0kuqqqNwDuA05NcQy9InreZ/d4GHJ7kauB6Hn0f/MeTXJvkOuAS4GrgInqXkloN\njgPvBcaSXJPkBuCIvm1nAv+RRy9TTdVfmhGnVZckdeIZhySpEwfHpSFL8k7gqE2aL62qI4dRjzQV\nL1VJkjrxUpUkqRODQ5LUicEhSerE4JAkdWJwSJI6+f8z+eDpswzpuQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x22e30783ac8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(train['interest_level']);\n",
    "pyplot.xlabel('interest_level');\n",
    "pyplot.ylabel('Number of occurrences');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_train = train['interest_level']\n",
    "X_train = train.drop(['interest_level'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 第一步：找出大约多少弱学习器合适"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xgb1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=1000,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#直接调用xgboost内嵌的交叉验证（cv），可对连续的n_estimators参数进行快速交叉验证\n",
    "#而GridSearchCV只能对有限个参数进行交叉验证\n",
    "def modelfit(alg, X_train, y_train, cv_folds=None, early_stopping_rounds=10):\n",
    "    xgb_param = alg.get_xgb_params()\n",
    "    xgb_param['num_class'] = 3\n",
    "    \n",
    "    #直接调用xgboost，而非sklarn的wrapper类\n",
    "    xgtrain = xgb.DMatrix(X_train, label = y_train)\n",
    "        \n",
    "    cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], folds =cv_folds,\n",
    "             metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)\n",
    "  \n",
    "    cvresult.to_csv('1_nestimators.csv', index_label = 'n_estimators')\n",
    "    \n",
    "    #最佳参数n_estimators\n",
    "    n_estimators = cvresult.shape[0]\n",
    "    \n",
    "    # 采用交叉验证得到的最佳参数n_estimators，训练模型\n",
    "    alg.set_params(n_estimators = n_estimators)\n",
    "    alg.fit(X_train, y_train, eval_metric='mlogloss')\n",
    "        \n",
    "    #Predict training set:\n",
    "    #train_predprob = alg.predict_proba(X_train)\n",
    "    #logloss = log_loss(y_train, train_predprob)\n",
    "\n",
    "   #Print model report:\n",
    "   # print (\"logloss of train :\" )\n",
    "   # print logloss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#params = {\"objective\": \"multi:softprob\", \"eval_metric\":\"mlogloss\", \"num_class\": 9}\n",
    "xgb1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=1000,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "modelfit(xgb1, X_train, y_train, cv_folds = kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1oAAANGCAYAAADktv9+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzs3XucXVV9///XZ87cZzKTZCZXEhIg\nEEhAUEQRqYIVaL3V3iz2Wy+V1rb219bW1odav4q2Vtv6Q/vrzbZWqbZfb63anyIGREEEhCKCJFzC\nJeSeSSaXud9nff/Ye2ZOhkkyCTuZXF7Px2M/ztlrr73P2uecJOedtfbakVJCkiRJklSciplugCRJ\nkiSdbAxakiRJklQwg5YkSZIkFcygJUmSJEkFM2hJkiRJUsEMWpIkSZJUMIOWJEmSJBXMoCVJkiRJ\nBTNoSZIkSVLBDFqSThkR8ZmIGIiIC6bY9p6ISBHx2knlTfm2eyNiX0QMRURbRHw7In41ImrK6i7P\nj1G+dEbEQxHxzogoHYvzPJiIeEdEvHWm2/FcRER9RFwfEVdMse2t+fu+fAbatThv10XH+rVnUkTc\nGBHdM90OgPz9TxHROtNtkSSDlqRTyTuBHcC/RUTVWGEevD4E3JhS+kZZ+dnAj4E/BX4AvBl4BfB7\nwFbgM8D7p3idvwVeki9vAO4CPgH8VfGndNjeAbx1phvxHNUDHwSumGLbTWTv+/Zj2aDcYrJ2nVJB\nS5I0tcqZboAkHSsppc6IuA64hSwgfTAPXJ8H2siCGAARUQl8HZgLvCil9Oikw305Ij4MPH+Kl9qU\nUvph2fq3I+J84I3Auwo7IT1LSmkXsGum21GkiKhPKfXOdDskSYfHHi1Jp5SU0neATwHvi4iLgeuB\nC4HrUkodZVV/HlgFfGSKkDV2rI0ppa9P86U7gKHygoioiIh3R8Rj+ZDGnRHxuYhYMnnniHhbPgSx\nPyL2RMTXIuK8SXXOjIgvRsS2/HhtEXHb2FC2iHgGWA28vGxo4zMHa3Re5+8i4k0R8WhE9ObteM00\nz7v8WNUR8f6y890VEZ+NiHmT6r0iIm6PiN0R0RcRmyLiv/Ihg8uZCFIfLDuPG/N9nzV0MD/W2oh4\nSUTcnR/zmYj49Xz7qyPigfzcHo6In5nUnhV5O5/I62yNiG+UD0HNhzH+T7762bJ2XV9W53URcU9+\njK6IuDUiXjLptcaGvr0gIv4zIvYCT+XbDvr5HuA9f2d+vBVTbPvLiBgcG2YXEc+PiG/m38OB/HVu\nmur7eCQi4vK8vV35e3B3RLz6APXuyb/rWyPizyLiNyZ/rs+xLedHxH9HxN78dR6MiLdMqlORf18f\nz78z+yLiJxHxB2V15kXEP0fE5rLv9F0R8coi2inpxGaPlqRT0Z8A1wD/CSwFPpVSunVSnavyx///\nCI5fEVmPGEAz8HPAzwB/OanePwJvB/4O+CawHPgz4IqIeEFKqR0gIt4L/AXwBeC9QAtZQLwnIi5J\nKT2RH+9bQAl4N7AJaAUuA2bn238+P+cOsiGEAAPTOJ9XA5cAHwC68+N/LSJWppSensb+REQF8N/A\nT5ENobwbWEY2ZPP2iHhhSqkv/yF9E3An8DZgH3Aa2ftXTTYk8GeAbwP/Cnw6f4lD9WItBD6bv/YW\nsuGfn4mIpcAvkb2/Hfk5fj0izkwpbcv3XQzsBt6Tv85c4C3AvRHx/JTS48ADwK/nr/Hn+TmQvxYR\n8avAf5D1pr4RqMnfx9sj4qdTSj+Y1N6vAl8k+0+BhrzsUJ/vVP6d7Hv3VsqGuUZ2veCvAd9IKbVH\nRANwK7AB+F2yHt6FwJXArIMcf1oi4uX58X8CXEf2vXsH8I2IeGNK6Ut5vefl9daTvce9wG/nbS1E\nRKwk+/7tBH6f7LP9NeDGiFiQUhob4vtusj9nfw58H6gCzmX/9/vzwAvIhhevz7e9gOzPqKRTXUrJ\nxcXF5ZRbyH7sJrIf7o1TbL85314zqTzI/pNqbCmVbVue7zPV8tlJdc/Ny/9+0vFflJd/JF+fTfZj\n86ZJ9ZYC/cB/5Ost+X5/cIjzXgvcfhjvUyK7rm1WWdkCYAR4z2Ec59r8WL8wqfyFefnv5Ou/mK9f\neJBjteZ1rp9i21vzbcvLym7Pyy4uK5sLDOfv7eKy8gvzur93kNcvkf3oXg/cMMW5vHVS/Qqya/p+\nAlSUlTeSBZq7ysquz4/xoUnHmNbne4D2/hewedJr/2x+vNfk6xfn6z93BMe/Eeg+RJ178nNtLCsr\nAQ/nbYu87MtkYb510vu3bvLneoDXGXv/Wg9S5wv5n52lk8q/BfQAzfn6N4AfH+L1uoBPHO575uLi\ncmosDh2UdMrJe1d+DxgF5pP9uJ6uPyAbAji2PDRFnb8h6wG6hKxH4H1kk2J8oazOlfnjjeU7ppTu\nAx4FfjoveglQN0W9zcB3y+rtIRti9icR8Uf5MLCi/o7/Xkqpq+y128h6A5YdxjFeQ9Y79Y2IqBxb\ngAfJgtwVeb0HgUHgnyPiLRFxZhEnAGxPKf1obCWltIfsHB5MEz1XkL33UHZueVvfFxGPRMQgWUAb\nBM4G9hu+eQAryXrFPp9SGi1rQzdZCLo0Iuon7fNfk9afy+f7WWAJUD6c7dfJ3veb8/Ungb3AX0bE\nb0fEqmke+5Dy3rIXA/+ZnzMAKaURsh6hJWTvEcDLge+mvDc3rzdKFsCK8grgtvzPULkbySZaGRvO\neR9wYUT8Q0RcExFNUxzrPuCt+RDDS6Nskh1JMmhJOhX9MdmPqV8FniAbQlY3qc6m/HFymPg/TISo\nBw5w/C0ppfvz5faU0kfJhgT+ckRck9cZG1o01ex428q2T6teSimRha41ZEOeHgB2RcT/FxHPdejX\n7inKBsgC4HQtIOudG2T/oDpENkStFSCl9BRZINgJ/D3wVEQ8VX5dzBHaM0XZ4OTylNJg/rS2rPgG\nss/v68BryULDJWQhezrvwaE+wwpgzqTy/eo+x8/35vx4Y9ekzQFeB3wuDzuk7PrEl5MF3b8A1uXX\naH2ogPAwh6wn+EDnD/t/39umqDdV2ZFqmWZbPkr2d8WlZO/h7vwasxeW7fMrwL8Bv0HWa7cnsuss\nFxbYXkknKIOWpFNK/j/1Hyb7kfklsqFmK4CPTKo6ds3W68oLU0o7x0IU2bCh6fpJ/jjWezYWXhZN\nUXcx0H6Y9UjZ5BzXpZQWkvUQfILsOpi/Pox2Hi3tZOdyyQGWsWvGSCndmVJ6Ldn1bZeS/YD9ZERc\ne6wbnfs1su/L+1JKa1JK9+Wf/3Tv1XSoz3CUrDepXJpc8Ug/37Keo9dHxGyy/2CoIevpKq/3cErp\nWrKgcRHwJbJr1p7rTJl7yc7xQOcP+3/fF0xRr8jgsns6bUkpDaeUbkgpvYBsqOkbyYbsrhnrgUwp\ntaeU3plSWk72nzLvBX6BST3Qkk5NBi1Jp4x8qNq/kf2Q+gOAlE3DfgPwBxHx0rLqXwMeIZud8NwC\nXn5sZrid+eN388f9LvKPiEvIhqPdlhfdA/RNUW8J+RCoqV4spbQ+pfTnZNfAvKBs0+H2RBXlm2Q/\n4EtlvX3ly+OTd0gpjaSU7iWbnAEmzmNsAo9jdR6JSZOG5LPlnTap3oHa9TjZNVq/GhFRdowGsmvS\n7kmHOX37QT7fA/ksWS/dG8n+c+GelNJjBzh2Sik9lFL6Q7LhntM5/sHa2gPcC/xCec9xPvTx18gm\nDFmfF98BvCLKbjic1/vl59KGSW7LX2PxpPI3k12z98PJO6SU9qWU/pOsl3Uu2fWYk+tsSin9Hdl/\n0jyn90zSycFZByWdSt5LNmHBz6aU9pWV/2+yIWGfiYiLUkp9KaWRiHg92VCt+yLiX8gmVdhLNgTu\nxWS9U1NN/X56RFyaP28gG6b4XmAj2WxypJQej4h/Bn4vIkbJhiYtJxuitpmst4KU0r6I+DPgLyLi\nc2TXebWQ3Ri3n2zWvrHZ2v4O+ArZcMhBsiD2POBjZW17GLg2In4FeBroTyk9fDhv4hH6IvC/gG9F\nxN+QXdsyRHZ9zpXAf6eUvhYRv523+yay4Zu1ZLMPAnwHIKXUFREbgZ+LiNvIhv+1p5SeOUpt/ybZ\ndTiPkfVMXkw2c+WWSfWeIgvF/ysiHiWb1GFbSmlbRLybbNbBb0bEP5H1KP0J2XfpPYdqwGF8vlNK\nKT0WEfeQfQ+Xks12WX7815D1jn2d7HsRZD0zs5no3T2YUkT80hTlPSmlm/PXvRX4XkR8PG//O4Dz\ngTfmQyMh61l+LXBbRHyE7P38bSZmXhxlel4bEc/qcc7D0ofIrhn8XmT3wttD9t18NfDufBglEfEN\nsslj7iebbXIZ2b32NgJPREQz8D2y4cSPkfVwX0I2K+ZXp9lOSSezmZ6Nw8XFxeVYLGShaBD45wNs\nv5RsJr0bJpU3kf1IvI+Je2G1kU3T/Q6gvqzucp4922AfWY/GJ4CFk45dQXa9zeN523aRTw4wRfuu\nI7smaICsl+HrwKqy7fPJei3GfuB35fXfyf6zHS4jC4+defueOcT7loC/m6L8GeDGw/wMKsmGoT2Y\nvy9deXs/Bawo+xy+mh+/n6z38XbgtZOO9dNk1yn15228MS9/K1PPOrj2AOfwzUOdM1nY+HT+ufeQ\nTT1/eX7c2yfte21+ToNMmhmRbJr/H+bn3k0WHC+btP/1TDFr3nQ/30O8/7+ZH7sXaJq0bSVZYHgy\n376PrBfqLdM47o0ceLbNZ8rqXU7Wm9Sdv8Y95LMeTjre5fn71E92LdVfkf05SeQzAh6kLdcfpC2p\nrN75ZLdu2Ef2Z+pBnj1b5B8Bd5H9uRwgC1ifBpbl22vIbtHwENnfDb1kget6yv5ecHFxOXWXselU\nJUmSjksRcQtZeD5nptsiSdPl0EFJknTciIgbgB+TDaGdSzas7yqyXl1JOmEYtCRJz0k+ycjBjKay\n+0dJh1Aimxl0IdmQv0eAN6WU/n1GWyVJh8mhg5KkIxYRy4ENh6j2oZTS9Ue9MZIkHUfs0ZIkPRfb\nyGZaO1QdSZJOKfZoSZIkSVLBjosbFkfEOyJiQ0T0R8SPIuKnDlL39ohIUyw3ldW5cYrtz7oBoSRJ\nkiQdDTM+dDC/aeYnye5HcxfwW8DNEbEqpbRpil1+AaguW28hu4fFVybV+zbw62Xrg4fRpgAWk92n\nRJIkSdKpbRbZTeinPRxwxocORsS9wAMppd8pK3sU+HpK6b3T2P+dZLMTLUop9eRlNwKzU0qvP8I2\nnQZsOZJ9JUmSJJ2UlqSUtk638oz2aEVENXAx8LFJm24BLpvmYa4DvjgWsspcERE7ye76fgfwpyml\nnQdoRw3ZHd73s3nzZpqamqbZDEmSJEknm87OTpYuXQqHOdptpocOtpLdL6NtUnkb2f0zDioiXgSc\nz7NvYngz2VDCjcAZwJ8B342Ii1NKA1Mc6r3ABycXNjU1GbQkSZIkHbaZDlpjJo9fjCnKpnIdsDal\ndN9+B0vpS2WrayPifrLQ9Wrgq1Mc56PADWXrs3DooCRJkqQjNNNBqx0Y4dm9V/N5di/XfiKiHrgW\n+MChXiSltD0iNgJnH2D7ADDe05XNhSFJkiRJR2ZGp3dPKQ0CPwKumrTpKuDuQ+z+BrLrqv79UK8T\nES3AUmD7ETRTkiRJkg7L8XAfrRuA34iIt0XEeRHxCeB04FMAEfG5iPjoFPtdRzYz4e7ywohojIiP\nR8RLImJ5RFwBfIOs9+xrR/VMJEmSJImZHzpISulLeY/TB4BFwFrgVSmljXmV04HR8n0i4hzgcuDq\nKQ45AlwAvBmYTdaL9T3gV1JK3hdLkiRJ0lE34/fROh5FRBPQ0dHR4ayDkiRJ0imss7OT5uZmgOaU\nUud09zsehg5KkiRJ0knFoCVJkiRJBTNoSZIkSVLBDFqSJEmSVDCDliRJkiQVzKAlSZIkSQUzaEmS\nJElSwQxakiRJklQwg5YkSZIkFcygJUmSJEkFM2hJkiRJUsEMWpIkSZJUMIOWJEmSJBXMoCVJkiRJ\nBTNoSZIkSVLBDFqSJEmSVDCDliRJkiQVzKAlSZIkSQUzaEmSJElSwQxakiRJklQwg5YkSZIkFcyg\nJUmSJEkFM2hJkiRJUsEMWpIkSZJUMIOWJEmSJBXMoHUc6x0cZvl7bmL5e26id3B4ppsjSZIkaZoM\nWpIkSZJUMIOWJEmSJBXMoCVJkiRJBTNoSZIkSVLBDFqSJEmSVDCDliRJkiQVzKAlp5GXJEmSCmbQ\nkiRJkqSCGbQkSZIkqWAGLUmSJEkqmEFLkiRJkgpm0JIkSZKkghm0JEmSJKlgBq0TREppppsgSZIk\naZoqZ7oBmp6rPvF9XnpWK5etaOGys1pZ0FQ70006Ir2Dw6z6wBoAHvnwNdRX+xWUJEnSycdfuSeI\nbfv6+cqPtvCVH20B4Kx5DVx2ViuXndXCpWe2MKeheoZbKEmSJGmMQesE8c9vupgfbdrLPU/t5uGt\nHTy1q4endvXw+R9uJALOW9jES/PerkvOmEtjjR+tJEmSNFP8NX6CuPzsVq5evRCAjt4hfrhhN/c8\ntZu7n2pnfVs3j2zv5JHtnfzLnRsoVQQXLmke7/F6wbI51FaVZvgMJEmSpFOHQesE1FxfxTWrF3JN\nHrx2dvXzw6f3cM9T7dz15G427enlgU37eGDTPv7ue09SXVnBC5fN4bKzWnjJWa08b0kzVSXnQZEk\nSZKOFoPWSWD+rFped+FiXnfhYgA27+nlnqezHq+7nmxnZ9cAdz+1m7uf2g2sp6G6xIvOmMtLV7Ty\nkrNaWDa3fmZPQJIkSTrJGLROQkvn1rN0bj1veOFSUko8tauHe55q5+6ndnPP07vZ1zvE9x7fxfce\n3wVAc13V+L7b9vWxYv6smWq6JEmSdFIwaJ3kIoIV8xtZMb+RN71kOaOjiUd3dHL3k9n1Xfdt2ENH\n39B4/Vfe8H3OP62Jq1ct5OrVC1i5YBYRMYNnIEmSJJ14DFqnmIqKYPXiZlYvbuY3X3YmQyOj/M+G\nPfzqp+8FIALWbu1k7dZObrh1Pcta6rl61QKuXr2QF5w+h1LFiRG6vF+XJEmSZpK/Pk9xVaUKLjp9\n9vj69//kCu55ag+3PLKD7z/RzsbdvfzLnRv4lzs30NJQzSvPW8DVqxfw0hWtzmQoSZIkHYBBS/tp\naazhDZcs5Q2XLKVnYJjvr9/FLY+0cdujbezuGeRL92/mS/dvpr66xBUr53H1qoVcuXI+zfVVhz64\nJEmSdIowaOmAGmoq+dkLFvGzFyxiaGSU+zbsYc26Hdyyro0dnf186+EdfOvhHVRWBJee2cLVqxdw\n1aoFLGqum+mmS5IkSTPKoKVpqSpV8NIVrbx0RSsfet1qHt7awS3r2rjlkR2sb+vmB0+284Mn2/nA\nf6/jwiXNXL16IVevWsCK+Y1OpiFJkqRTjkFLhy0ieN6S2TxvyWz++JqVbGjv4dZHdrBmXRsPbNrL\nQ1s6eGhLB3+95nHOaG3IJ9NYwPOXzpnppkuSJEnHhEFLz9kZrQ28/WVn8faXncXOrn5ue3Qnt6zb\nwV1P7mZDew//9P2n+afvP01rYw1XnjtvppsrSZIkHXUGLRVq/qxa3vii03nji06nq3+IO9bv4pZ1\nbXzvsZ20dw/wlfu3jNf94688xKsuWMQVK+fTWONXUZIkSScPf93qqJlVW8VrnreY1zxvMYPDo/zw\n6d186+HtfPF/NgOMT6ZRXVnB5StauWb1Al553gJaGmtmuOXT5/26JEmSNBV/FR7H6qsreeZjr57p\nZhSiurKCl50zjxcunzMetK67fDm3PbqTZ3b38t3HdvLdx3ZSEQ9zyfK5XLN6IVevXsCSOfUz3HJJ\nkiTp8Bm0NGPedfVK3v/qVaxv62bNuh2sWbeDdds6uXfDHu7dsIcPf/MRzj+tiWtWLeSa8xdytjMY\nSpIk6QRh0NKMighWLpzFyoWz+P2fPpvNe3q55ZE21qzdwf9s3MParZ2s3drJ/3vres5obeCa1Qu5\nZvUCLlwym4oKQ5ckSZKOTwYtHVeWzq3nusvP4LrLz6C9e4DvPNLGmrIZDD91x1N86o6nWNBUw9Wr\nFnLN6oW8+My5VJUqZrrpkiRJ0jiDlo5brY01XPui07k2n8Hw9sd38e11O7j9sZ20dQ7w+R9u5PM/\n3EhzXRU/fd58rlm9kJedPY+66tJMN71wTrohSZJ0YvHXmk4Is2qreO2Fi3nthYvpHxrh7qfaWbO2\njVsfbWNPzyBffWArX31gK7VVFbz8nHlcuXL+TDdZkiRJpzCDlk642Q1rq0q84twFvOLcBfzFaOL+\nZ/bw7XU7uGVdG1v39bFmXRtr1rWN1//ifZt49fMWs6CpdgZbfWKw50ySJKkY/orSCa1UEbz4zBZe\nfGYLH3jNKtZt62TNuh3cvHYHT+7sBuDD33yUD3/zUZ5/+ux8Mo2FnNHaMMMtlyRJ0snMoKWTRkRw\n/mnNnH9aM79zxVnjPTMXLmnmoS0d/HjTPn68aR8fu/kxzlnQOB66Vi9uctp4SZIkFcqgpZPeF95+\nKV39w9zySBu3rNvBPU/tZn1bN+vbnuRvv/skp82u4+rVC7hm9UIuWT6XktPGS5Ik6TkyaOmUsKCp\nljdduow3XbqMjt4hvvt4G2vWtnHH+l1s3dfHZ+96hs/e9QxzG6p5ZT6D4UtXtFJbdfLNYChJkqSj\nz6ClU05zfRU///wl/Pzzl9A3OMKdT+xizbo2bnssm8Hwy/dv4cv3b6GhusQVK+dz9eoFXHnufJpq\nq2a66ZIkSTpBGLR0SqurLnH16oVcvXohwyOj3LdhD2vW7eCWR9rY3tHPTQ9v56aHt1NVCi47q5Vr\nVi/kqlULmDerZqabLkmSpOOYQUvKVZYquGxFK5etaOX6163mJ1s6WLNuB2vW7eCpXT3csX4Xd6zf\nxZ9+/WEuPn1OdoPkc1pnutknJKeRlyRJJzt/3UhTiAguXDqbC5fO5t0/cy5P7uzOerrW7eChLR3c\nv3Ev92/cy0e+NbHPzWt38KLlc1kyp85ZDCVJkk5xBi0dUyfazZHHrJjfyIr5K/jdK1ewvaOPW9a1\nsWbdDu7dsIeR0QTAu778EACtjdVcuGQ2F+VB7cIls2mu9/qumWDPmSRJmin+6pAO06LmOt5y2XLe\nctlytu3r5bKPfQ+A1YubeHxHF+3dg9z22E5ue2zn+D5ntjZw4dKJ8HXeolnUVDqjoSRJ0snKoCU9\nB7Prq8eff+W3X0JFBOu2dfLQ5n08uHkfD23Zx8bdvTzd3sPT7T187cdbAaguVXDe4iYuWtLMRadn\nvV5ntDY45PAEZc+ZJEmazF8DUoFqq0pcvGwOFy+bM162t2eQB7fsmwhfm/ext3eIh/Ln/3bPRgCa\n66p43pJmnj825HDpbFobnd1QkiTpRGTQko6yOQ3VXLlyPleunA9ASolNe3p5sCx4rd3WSUffEHc+\n0c6dT7SP77tkTh0XLZ3NqkVNM9V8SZIkHQGDlnSMRQTLWhpY1tLAz110GgCDw6M8vqOLB7fs48FN\n2ZDDJ3d2s2VvH1v29vHNn2wf3/+aT3yflQubOHfhLFbmyxmtDVSVKmbqlCRJkjSJQUs6DlRXVnDB\nkmYuWNLMmy5dBkBn/xAPb+ngwc37+NHGvXw3n1xj894+Nu/t4zuPto3vX1UKzprXyMqFszhnwSzO\nzR+dal6SJGlmGLSk41RTbRUvXdHKS1e07jfZwmff+kKe2d3L+rYuHtvRxfodXfQMjvDYjmy9XGNN\nJWcvaBwPXisXzuLchU3Mbaie6iV1nHPSDUmSThz+K62T0ol6v67pePGZLVx57oLx9dHRxNZ9fTy+\no4vH27p4fEcX69u6eGpXN90Dw/x40z5+vGnffsdobawZD1/nLpzFOQtncc6CxmN9KpIkSSctg5Z0\ngquoCJbOrWfp3HpeuWoigA0Oj/LM7p7xXq/H8gC2aU8v7d0D/ODJAX7w5MTEGxGwZHbd+Pptj+7k\n4mVzWNRc6/DDU8yx7Dmzl06SdLLyXzTpJFVdWcE5C7JeKy6cKO8ZGGZ9W9fE0MO8F6y9e5DNe/vG\n6/3eF34MQEtDNatPa+b8xU1ccFoz55/W7LVfkiRJh2DQkk4xDTWVPP/0OTz/9Dn7lbd3D/CTLft4\n2433A7ByQSNP7uphd88g31+/i++v3zVet7muivNPa+L8xc3jIWx5SwMVFYYvSdLxz950HQt+qyQB\n2XVbl57ZMr7+td99KRURPLaji7VbO1i3rYOHt3bw+I4uOvqGuOvJ3dz15O7x+o01laxanIWvC5Zk\nj2fOa6Rk+JIkTcPJGH5OxnPS9PlpSzqg2qoSFy2dzUVLZ4+XDQ6Psr4tC19rt3Wwdmsnj27vpHtg\nmPs27OG+DXvG69ZVlThv0SwuOG2s56uZs510QzPAHzsCvweSji3/hpF0WKorKzg/v1ZrzPDIKE/u\n6mbt1s4sgG3t4JHtnfQOjvDApn08UDbrYXVlBSvLwtZ3Hm1jeUsDC5traW2ocfihNA3HKjCcbK+j\nI+PnIx0Z/6RIes4qSxWcu7CJcxc28UsXLwFgZDSxob1nPHit3dbBuq2ddA0M8/DWzvF9f/8LD44/\nryoFC5pqWdRcy8LmOhY1144vY+utjTUOR9Rxyx+kAgOqjj2/C8cnPwVJR0WpIlgxv5EV8xt5/fNP\nA7J7fm3a08uPNu3lXV9+CIALTmtmZ1c/O7sGGBpJbNnbx5a9fcDeKY9bWZGFsYXN2bIof754dl22\n3lzLvMYaKksVx+pUJUmSnsWgJT0HJ/ONkY+GiopgeWsD85tqxoPWl37rUuqrKxkaGWVn1wA7OvrY\n3tHPjo7+8cdtHX3s6OinrbOf4fwGzVv39R34dQLmz6plflPNeNk/3fE0i2fX0tJQQ+usGlobq2lt\nrKG2qnTUz1uSJJ16DFqSjgtVpQpOm13HaWU3TZ5seGSU9u5BtufBa3tHPzs6+9m2b2J9LIzt6My2\njfmb256Y8pizaippyUNXa2MNrbOy5y2NNczbr7yGhuqS9w+TJEnTYtCSdMKoLFWMDxk8kNHRRHvP\nADs6+nlmd8/4NWC/8ILT2Nc7RHv3AO1dA7R3DzI4MkrXwDBdA8M8s7v3kK9fU1kxHrrmNVbnvWPV\nNNVVjdfZ2dnPspYGA5kkSafCiAfWAAAgAElEQVQ4g5akk0pFRWTDBmfVsmL+xOyGf/768/e7ODil\nRGf/MLu7s9DV3j0wEcJ6BvMwNrGtd3CEgeHRQw5bvOLjd1BXVWJZSz1ntDawrKWBM1rr88cG5s+q\nMYRJkk5ITrpxeHx3JJ2SIoLmuiqa66o4c96h6/cODrO7e5BdZT1i7d0D7O4eYEdnP2vWtQHZ9WF9\nQyM8tqOLx3Z0Pes4YyFseUsDy1sbWN6yfwhzentJ0qnuZAl0J2arJekYq6+upH5uJUvn1j9rW/k/\nCA/876vY0zPIxt29bGjvYePuHjbs7mXj7h627O07aAirrapgeUtDFsRaG7Iw1tLA8tZ6Fsw68HBJ\nSZJ0/DFoSScAZzc8cVRXVnDmvEbOnNfIlZO2DeZDD59p7+GZ3T35Yy/P5CGsf2j0oCFs6ZyJkPf5\nH25k2dwGFs+uZVFzHS0N1faGSZJ0HDFoSdIxUl1ZwRmt2TDByYZGRtm6t48Nu3vYWBbAnmnvYXMe\nwp7Y2T1e/6Pfemz/Y5cqWNBcw6LmOhY317Jo9tgNn7PHxbPrmFNf5fVhkiQdIwYtSToOVJUqsuGC\nrQ2wcv9tYyHs8bZOfuvzDwBw9aoF7OwaYHtHHzu7BhgcGWXznj427znwRB01lRUT4Wt2LYvzx7Gy\nxc11NNX5z4IkSUXwX1RJOs6NhbDyGzB/8tqLxi8OHhoZpa0zu4/Y9o5+tu/Lbvq8LX/c3tFPe/cA\nA8OjeU/Zgaeyr68usbBp4nqwj9z0KM11VTTUVFJXVaKhpkR9deX4Y331s9erShVH782QJOkEYdCS\npBNcVamCJXPqWTLn2RN1jBkYHqGtY4BtHX1s78gD2L5+tnf0sS1/3Ns7RO/gCE+394zv9x/3bjrs\n9lSXKqivKdFQXUlddYmGKcLY2HpVaWIo4wMb93L63CxQ1laVDvt1JUk6nhi0JOkUUFNZ4vSWek5v\nOXAY6x8aYXtHPxvau3nbjfcD8FsvO5PBkVF6B0boGRymbzB77B0coWdgbD17PjyaABgcGWWwd5R9\nvUOH1cZf+9f7xp831VYyv6mW+bNqsmXs+aSyxhr/GZMkHZ/8F0qSBEBtVYkzWhtYUDZE8Q9eefa0\n718yODxaFsSG6dkvnI3QOzBMz+AIfYPD4+sd/UN8/cfbAFgyp45dXdkQx87+YTr7u3mybAKQqdRX\nl/LgVcv8pvLH/Z+X95ydCFJKDAyP0jOQvY/dA8P0DA7TPTBMd/8we3sHx+t+9YGtzKmvznsK817D\nmrLnx+lwztHR7Bz7h0bY1zdxPu3dA7Q0ZPeccyZNSScyg5akcU4jr+eiurKC6soKmuurpr1P7+Dw\neNC65Q9fRl1Vic7+YXZ19dPWOcDOrn52dg6wsytfOvvHH3sGR+gdHDnkdWdjbRvzi/94N3VVpby9\nJapLFdRUVVBTytpfk59HdWUF1aUSNVUVVJcmymrK60zanlIaf52fbNnHyChZSMqX7oGR/HF4vHxi\n+8h+5WM9hIfy/q+vPWSd6lLF+DDOuuryIZwl6msqqa8qZdfhjdeppKG6RKks6Ny8dgcpZT2fYwFp\nYHiUgUnr/UMjz6oz8TjKwPAIA0OjDI6MTtnWl/3V7ePPa6sq9mtrXXXW1vrx88jOpbzd43WrJrbV\nV2dDWYnpvaeSVASDliTpuBERNNdV0VxXxYr5sw5at2dgeL/w1dbZz66xQJYHtLbOfjr7hxkcnvhR\n/+j2Z9+n7Gi49p/vLeQ4dXkImlWbXdfWUJ1NTHL7+l0AvOzsVvrLehP78qGcvYMj+w/n7Bulo+/w\nhnOWe9eXHyrkfKZSWQqGR7K2RsBYXu0fGqV/aJA9PQfZ+Qi94uN3sKCphnmzallwgN7QloZqKo/D\n3kBJJwaDliTphNRQU8kZNZVT3pesXP/QCJv29HD1J+4E4FO/9gIigoHhUQbHl5GJ9ZHscSBfxsoG\nhkb22zY4xbaB4VG6+ocBWNRcm4ejShprKmmortw/ME0qb6yppLG2ksZ8W0O+rTTF8LnewWFWfWBN\ndj5vuviAwzv3H845Qu/kx4HseTakc/+hnn2Dw3T1D3Pvhj0AXLJ8DvXVldRUVlBbVRp/rK2qoKZy\n0mPZ9mfX37+sprKCwZHR8fNZe/3VVETFeBv7hiauB+wdHKF3KGtb79j6WL3x9RH6hsrOb2h4v21j\ndnT2s6OzH+g44HenIqClsWbiusCxMDbpWsF5jTX79ZpKEhi0JM0AhyjqWKqtKu03I+PLzpk37evO\nDld5ALrtXS8/aq8zXUcynLNc+fn829tedNTOp3wYYURQlw8NbCn4dXoGhlj9wVsA+OLbX0xn3/D4\nsNRdY72g+WN79wCjCXZ1DbCra4B1hzj2nPoqFjTVMm9W1hM25p/ueJoIGBlNDI8mhkdG88dsfWR0\ntOx5YmhkNHuctG1y3eGRUYZGJoZCXvnx2ylFEBFUVEBFBBURRIw9h6BsPa8T+bbxOvutx349jAB/\n8pWHaKqrnvgPgfw/ChpqSjTWVFJfnf8Hwth6PjTV6+10KjJoSZKkU0LExI/95y2ZfdDgODKa2N0z\nwM7OLGi1jV0fWHbd4K58fWgksbd3iL29Qzy2Y/+hqX9z2xNH7XzKtXUOHJPXuenhHYe9TwTj1wGO\nhbKGPJDV1+S9uNWV+/UKfvWBrTTWVFJZCqpK2XWQY8+zZXrPyz9z6VgzaEmSJE1SqohsqOCs2oPW\nGx1N7Osb2i+Abd3byye+kwWsX3zBadRUlaiqCEoV2Y//UkVQWaqgsiJ7npWVbasIKiuyYFFZUTFR\nViovD4ZGRnnjv2TXAv7n77yEmlKJ0ZTyJZu9cjQxXpbGn5OvJ0ZHOWT9/qER3ve1bNKVd1+zMrvl\nw+DIfhO59JTNjNk7NunL4DApZT1iPfmQVLqmFwinM8nLdFRWZMGrshRU5wFs7P0b84dfepD5s2qZ\n21BNS2M1LQ0148/nNlQzp756yiG80qEYtCRJko5QRUUwtyH7QX7uwqysd3B4PGj92evPP6pDVces\nWtR0VF9nLGi99aXLp/06KSX6h0b3m2Fz7B584yEtX+8ZGKajb4gv/s9mIJvkJZFdZziUD5PMHrPn\nwyOjDOZlY8Mop5rJMht2OQIHmQdmzbq2g55HBMypzz7jlrIANrehhtbx51lAa2k0mGmCQUvSSctr\nwSRp5pRfbzdvVs0h6/cODo8HrYNN8nIgKY1d55aFruEpAtrY867+Yd78mewm6e991bl09w+zu2eQ\nPd2D7O4ZyJ73DLKvd4iUYE++/uS0zhtm11WN94aN+cwPNrCspYHFs2tZ1FzH/Fk1zmp5kjNoSZIk\n6YQXkQ+vLEEdpYPWLe8NfNOlyw4Y6oZGRtnbm4WsPd2DtPcMsqd7gD09g+zuGWR392D+PCvbmwez\nsWv2YOLeBB+/Zf1+x86Gp9aweHYdi5prxx8XNdeNh7GWhmonEjmBGbQkSZKkKVSVKqZ1rd6Y4ZFR\n9vYOjYev7fv6eNdXfgLAqy9YyM6uAbbt66ets5/h0cT2jn62d/Qf8HjVpQoWNtfuF8QWz54IYoub\n62iq8+f88eq4+GQi4h3AnwCLgHXAO1NKdx6g7u3Ay6fY9K2U0qvzOgF8EHg7MAe4F/jdlNKhZmeV\npMPmEEVJEkBlqYJ5s2ryoZKz6B0cHg9af/3LF473nI2MJtq7B9i2r4/tHf37PW7r6Gf7vj52dQ8w\nODLKpj29bNrTe8DXrK8usbBpIgjecOt6FjfXMS+/19tYexprKp2F8Rib8aAVEb8CfBJ4B3AX8FvA\nzRGxKqW0aYpdfgGoLltvAR4CvlJW9m7gj4C3AuuB9wO3RsTKlNL+865KkiRJx1CpIljQVMuCplqe\nf4A6g8OjtHX2571efWzbt//j9o5+9vQM0js4wtPtE0MUP33nhimPV1dVelb42v/5xH3gvHasGDMe\ntMgC0b+mlD6dr78zIq4Bfgd47+TKKaU95esRcS3QSx608t6sdwIfSSl9NS97C9AG/CrwT5OPGRE1\nQPlVmrOe4zlJUuHsOZOkU0d1ZQVL59azdG79Aev0D42wvaOfDe3dvO3G+wF406Wns6d3iF1dA7Tn\nN+TuHhimb2jkkL1jkE3m0dJQTWtjDfObapnXOBHKmsuGKW7b18e8WbU01lQ6y+IBzGjQiohq4GLg\nY5M23QJcNs3DXAd8MaU0FuXPABbmxwAgpTQQEXfkx3xW0CILdB88jKZLkiRJM6q2qsQZrQ0saJro\nL3jvq8571uQevYPD7Mpvsr2ra/8bbu/qGmBXd3Zz7vbuAUYTtHcP0t49+KwbcJd75Q3fH39eX12i\nsaaSWbWVNNZWMasmuyF1Y23lRHlNJbNqq2isrcy252Xl20+2nrSZ7tFqBUpkvU3l2sjC0kFFxIuA\n88nC1pix/aY65rIDHOqjwA1l67OALYd6fUmSJOl4V19dybKWSpa1NBy03shoYm/vIDs7x8JXP7u6\nJ8JZW0c/92/cC2Q9boPD2b3LegdH6B0cYec0b0h9IHVVJRprK2monpg1cnQ0PadjzqSZDlpjJr+D\nMUXZVK4D1qaU7nsux0wpDQDj3wwvFJQkSdKpplQRtDbW0No49X3PegeHWfWBNQA8+IGrKFUE3f3Z\nDai78sfx9YFhuvqHxte7+7OyifpD4/sN5IGtb2iEvqERdpW95ok8vf1MB612YIRn917N59k9UvuJ\niHrgWuADkzbtyB8XAtsP55iSJK8FkyRNT01liZrGEi0HCGbTNTg8Ss/ARGBr7+7nzZ/5n4JaOXNm\ndCBkSmkQ+BFw1aRNVwF3H2L3N5BNYPHvk8o3kIWt8WPm14K9fBrHlCRJknQMVVdWMKehmqVz61m1\nuIkXLp87000qxEz3aEF2bdTnI+J+4B6ye1+dDnwKICI+B2xNKU2egfA64Osppd3lhSmlFBGfBN4X\nEU8ATwDvI5uZ8P8c1TORJB0We8+OzMn2vp1s5yNJcBwErZTSlyKihWwI4CJgLfCqlNLGvMrpwGj5\nPhFxDnA5cPUBDvtXQB3wD0zcsPhq76ElSaemY/VD3sAgSRoz40ELIKX0D2ShaKptV0xRtp5scosD\nHS8B1+eLJEk6AgZHSTpyx0XQkiRJOtoMjpKOpZPrrmCSJEmSdBywR0uSJOkEZA+ddHwzaEmSJBXI\nACQJDFqSJEk6CIOjdGS8RkuSJEmSCmbQkiRJkqSCGbQkSZIkqWBeoyVJkqQZ57VgOtnYoyVJkiRJ\nBTNoSZIkSVLBHDooSZKkU4rDFHUs2KMlSZIkSQUzaEmSJElSwRw6KEmSJB0FDlE8tdmjJUmSJEkF\nM2hJkiRJUsEcOihJkiSdwI7VEEWHQh6eSCnNdBuOOxHRBHR0dHTQ1NQ0082RJEmSNEM6Oztpbm4G\naE4pdU53P4cOSpIkSVLBDFqSJEmSVDCDliRJkiQVzKAlSZIkSQUzaEmSJElSwQxakiRJklQwg5Yk\nSZIkFcygJUmSJEkFM2hJkiRJUsEMWpIkSZJUMIOWJEmSJBXMoCVJkiRJBTNoSZIkSVLBDFqSJEmS\nVDCDliRJkiQVzKAlSZIkSQUzaEmSJElSwQxakiRJklQwg5YkSZIkFcygJUmSJEkFM2hJkiRJUsEM\nWpIkSZJUMIOWJEmSJBXMoCVJkiRJBTNoSZIkSVLBDFqSJEmSVDCDliRJkiQVzKAlSZIkSQUzaEmS\nJElSwQxakiRJklQwg5YkSZIkFcygJUmSJEkFM2hJkiRJUsEMWpIkSZJUMIOWJEmSJBXMoCVJkiRJ\nBTNoSZIkSVLBDFqSJEmSVDCDliRJkiQVzKAlSZIkSQUzaEmSJElSwQxakiRJklQwg5YkSZIkFcyg\nJUmSJEkFM2hJkiRJUsEMWpIkSZJUMIOWJEmSJBXMoCVJkiRJBTNoSZIkSVLBDFqSJEmSVDCDliRJ\nkiQVzKAlSZIkSQUzaEmSJElSwQxakiRJklQwg5YkSZIkFcygJUmSJEkFM2hJkiRJUsEMWpIkSZJU\nMIOWJEmSJBXMoCVJkiRJBTNoSZIkSVLBDFqSJEmSVDCDliRJkiQVzKAlSZIkSQUzaEmSJElSwQxa\nkiRJklQwg5YkSZIkFcygJUmSJEkFM2hJkiRJUsEMWpIkSZJUMIOWJEmSJBXMoCVJkiRJBTNoSZIk\nSVLBDFqSJEmSVDCDliRJkiQVzKAlSZIkSQUzaEmSJElSwQxakiRJklQwg5YkSZIkFcygJUmSJEkF\nM2hJkiRJUsEMWpIkSZJUMIOWJEmSJBXMoCVJkiRJBTNoSZIkSVLBDFqSJEmSVDCDliRJkiQVzKAl\nSZIkSQUzaEmSJElSwQxakiRJklQwg5YkSZIkFcygJUmSJEkFm/GgFRHviIgNEdEfET+KiJ86RP3Z\nEfH3EbE93+fRiHhV2fbrIyJNWnYc/TORJEmSpEzlTL54RPwK8EngHcBdwG8BN0fEqpTSpinqVwO3\nAjuBXwK2AEuBrklV1wGvLFsfKb71kiRJkjS1GQ1awB8B/5pS+nS+/s6IuAb4HeC9U9R/GzAXuCyl\nNJSXbZyi3nBKyV4sSZIkSTNixoYO5r1TFwO3TNp0C3DZAXZ7HXAP8PcR0RYRayPifRFRmlTv7IjY\nlg9J/GJEnHmIttRERNPYAsw6glOSJEmSJGBmr9FqBUpA26TyNmDhAfY5k2zIYAl4FfDnwLuAPy2r\ncy/wZuAa4DfzY90dES0Hact7gY6yZcvhnIgkSZIklZvxyTCANGk9pigbU0F2fdbbU0o/Sil9EfgI\n2VDD7GAp3ZxS+q+U0sMppe8Ar843veUgbfgo0Fy2LDn805AkSZKkzExeo9VONknF5N6r+Ty7l2vM\ndmAopVQ+ucWjwMKIqE4pDU7eIaXUExEPA2cfqCEppQFgYGw9IqZ3BpIkSZI0hRnr0cpD0Y+AqyZt\nugq4+wC73QWsiIjydp8DbJ8qZEF2/RVwHllIkyRJkqSjbqaHDt4A/EZEvC0izouITwCnA58CiIjP\nRcRHy+r/I9AC/E1EnBMRrwbeB/z9WIWI+HhEvDwizoiIFwP/CTQB/3aMzkmSJEnSKW5Gp3dPKX0p\nn6TiA8AiYC3wqpTS2JTtpwOjZfU3R8TVwCeAnwBbgb8B/rLssEuAL5BNtrEL+CFwadkxJUmSJOmo\nipQONO/EqSuf4r2jo6ODpqammW6OJEmSpBnS2dlJc3MzQHNKqXO6+8300EFJkiRJOukYtCRJkiSp\nYAYtSZIkSSqYQUuSJEmSCmbQkiRJkqSCGbQkSZIkqWAGLUmSJEkqmEFLkiRJkgpm0JIkSZKkghm0\nJEmSJKlgBi1JkiRJKphBS5IkSZIKZtCSJEmSpIIZtCRJkiSpYAYtSZIkSSqYQUuSJEmSCmbQkiRJ\nkqSCGbQkSZIkqWAGLUmSJEkqmEFLkiRJkgpm0JIkSZKkghm0JEmSJKlgBi1JkiRJKphBS5IkSZIK\nZtCSJEmSpIIZtCRJkiSpYAYtSZIkSSqYQUuSJEmSCmbQkiRJkqSCGbQkSZIkqWAGLUmSJEkqmEFL\nkiRJkgpm0JIkSZKkghm0JEmSJKlgBi1JkiRJKphBS5IkSZIKZtCSJEmSpIIZtCRJkiSpYAYtSZIk\nSSqYQUuSJEmSCmbQkiRJkqSCGbQkSZIkqWAGLUmSJEkqmEFLkiRJkgpm0JIkSZKkghm0JEmSJKlg\nBi1JkiRJKphBS5IkSZIKZtCSJEmSpIIZtCRJkiSpYAYtSZIkSSqYQUuSJEmSCmbQkiRJkqSCGbQk\nSZIkqWAGLUmSJEkqmEFLkiRJkgpm0JIkSZKkghm0JEmSJKlgBi1JkiRJKphBS5IkSZIKZtCSJEmS\npIIZtCRJkiSpYAYtSZIkSSqYQUuSJEmSCmbQkiRJkqSCGbQkSZIkqWAGLUmSJEkqmEFLkiRJkgpm\n0JIkSZKkghm0JEmSJKlgBi1JkiRJKphBS5IkSZIKZtCSJEmSpIIZtCRJkiSpYAYtSZIkSSqYQUuS\nJEmSCmbQkiRJkqSCGbQkSZIkqWAGLUmSJEkqmEFLkiRJkgpm0JIkSZKkghm0JEmSJKlgBi1JkiRJ\nKphBS5IkSZIKZtCSJEmSpIIZtCRJkiSpYAYtSZIkSSqYQet4NtgD1zdny2DPTLdGkiRJ0jQZtCRJ\nkiSpYAYtSZIkSSqYQUuSJEmSCmbQkiRJkqSCGbQkSZIkqWAGLUmSJEkqmEFLkiRJkgr2nINWRDRF\nxOsj4rwiGiRJkiRJJ7rDDloR8eWI+H/y53XA/cCXgZ9ExC8W3D5JkiRJOuEcSY/Wy4A78+c/DwQw\nG/h94P0FtUuSJEmSTlhHErSagT35858B/iul1AvcBJxdVMMEjI5MPL/twzA6OnNtkSRJkjRtRxK0\nNgMviYgGsqB1S14+B+gvqmECouzjufdT8NXfgOGBmWuPJEmSpGk5kqD1SeA/gC3ANuD2vPxlwMPF\nNEsAREw8r6iEtf8F//6L0N8xc22SJEmSdEiHHbRSSv8AvAR4G3B5SmlsPNvTeI3W0fMrn4fqWfDM\nnfCZn4XObTPdIkmSJEkHcETTu6eU7k8pfS2l1B0RpYi4CLg7pXRXwe3TmDNeDr/+LWhcADvXwaev\ngp2PznSrJEmSJE3hSKZ3/2REXJc/LwF3AA8AmyPiimKbp/0seh5cdyu0ngOdW+Az18AzZltJkiTp\neHMkPVq/BDyUP38tcAZwLtm1Wx8pqF06kDnL4G1rYOml2bVan389rPvaTLdKkiRJUpkjCVqtwI78\n+auAr6SU1gP/ClxQVMN0EPVz4c1fh3NfAyOD8JVfhx/+40y3SpIkSVLuSIJWG7AqHzb4M8B38vJ6\nYOSAe6lYVXXwhs/BJb8JJPj2e+CW93uvLUmSJOk4cCRB67PAl4G1QAJuzctfDDxWULs0HRUleNVf\nwyuvz9bv/lvvtSVJkiQdByoPd4eU0vURsRZYSjZscOxX/QjwsSIbp2mIgMv/EGYthv9+R3avre6d\ncO1/QG3zTLdOkiRJOiUddtAC+L/s3Wd4VWX+vv3zTkIogQQEUVGxIGKjSVFRLCgWQMcOdgZFBQR1\nLDODzzj8ZmwzY0NAEUWxgdh7QQQUUYpUG9gRBQuiCYQSynpeLPgHKZqEJCvl/BzHfcwua6995Y3j\n5Vr7e0dR9NQWXnto2+OoyJp3hZrbw+jz8/faOvcpyGzwx5/Ny4Wb1h/XfyGkZ5RsVkmSJKmCK9I+\nWiGEI0IIL4YQPg8hfBZCeCGE0L64w6mQGnVwry1JkiSpDCjKPlrnEg/AWA7cBQwGVgBvhhDOLkqI\nEELvEMJXIYSVIYTpf1TaQgi1QwhDQgiL1n/mkxBCp205Z5mUngEDsuNV0KtM7rUlSZIkJa4oV7Su\nA66NoqhrFEV3RVE0MIqirsDfgH8U9mQhhK7k78HVEpgIvBpCaLiV49OJB3DsTrynVxOgJ/BdUc9Z\n4bjXliRJkpSoohStPYEXt/D6C8SbFxfWX4DhURTdH0XRJ1EUXQEsAHpt5fgewHbAyVEUTYqiaH4U\nRe9EUTR7o2MKe86Kx722JEmSpMQUpWgtAI7ewutHr3+vwNZfnWoFjNnkrTFAu6187CTgPWBICOGH\nEMKHIYT+6/f1KtI5QwhVQwiZGxZQqzB/R5nlXluSJElSIooydfA24K4QQgvgXeK9tA4DugOXF/Jc\n9YBU4k2QN/YDsONWPrMn0AF4DOgENAaGEP8t/yriOf8O/LOQ2cuHDXttZe0MYwfEe23lLIST74G0\nqkmnkyRJkiqkouyjdU8I4XvgKuDM9S9/AnSNouj5IuaINnketvDaBinAj8DFURStBaaHEBoA1xAX\nraKc82bg9o2e1wK+LUDu8sG9tiRJkqRSVdR9tJ4FfjNdIYRQJYTQMIqibwpxqsXEGx1veqWpPptf\nkdpgEbB6fcna4BNgx/W3DRb6nOs3Xd6w8TIhhILmL1+2tteWZUuSJEkqVkXaR2sr9gO+KswHoijK\nA6YDHTd5qyPxbYlbMgnYK4Swcfa9gUVRFOUV8ZyVx5b22vppXtKpJEmSpAqlOItWUd0OXBRC6BFC\n2DeEcAfQEBgKEEJ4OIRw80bH3wPUBQaGEPYOIXQG+hP/TqtA56z0Nt1r65GTk04kSZIkVShFunWw\nOEVRNDqEUBe4HtgJ+BDoFEXR/PWHNATWbXT8ghDCscAdwBzi/bMGAv8pxDm1Ya+tUWfBgslJp5Ek\nSZIqlBBFW5sPUcgThdAcmBFFUWqxnDBB60e8Z2dnZ5OZmZl0nJK1ekW8x9anr8bPj7oODr8mHqAh\nSZIkVXI5OTlkZWUBZEVRlFPQzxX4ilYIodkfHNKkoOdSGVKlOpw6DG7ZNX4+/kZY/BmcdFf8niRJ\nkqRCK8ytg7OIx6Nv6VLHhteL5/KYSlfKRhchQyp88AT8/Bl0GwmZDZLLJUmSJJVThRmGsQfxZsF7\nbGHtudH/qjw763Govh0snAnDjoQF05JOJEmSJJU7BS5aURTNL8gqybAqBbsfChePh/r7wbIfYEQn\nmDUy6VSSJElSuVIWxrurrKmzO1w4BvbpAmvz4Lle8Pp1sHZN0skkSZKkcsGipS2rWgvOfASO+Gv8\n/L3BMPIMWPFLsrkkSZKkcsCipa1LSYGj+sMZD0GVGvDFOLjvaPhpXtLJJEmSpDLNoqU/tv/J8ebG\nWbvCki/g/mPg0zFFO1deLgzIildebvHmlCRJksoIi5YKZqdm0HM8NGwHq3Jg5Jnwzh1QTBteS5Ik\nSRVJYfbRAiCEMJMt75cVASuBz4ERURSN38ZsKmtqbg/nPw+vXgvTH4SxA+CHj+CkQW5uLEmSJG2k\nKFe0XiPeLysXGA9MAJYBjYBpwE7A2BDCn4opo8qStHQ48U7ofBukpMEHT8KDJ0D2d0knkyRJksqM\nohStesBtURS1j6LoqiiK/hJF0eHArUBGFEXHAjcA/yjOoCpj2lwE5z2Xv7nxfUe5ubEkSZK0XlGK\n1pnAqC28/vj691j/fuQCrqUAACAASURBVJOihlI5sUf79Zsb75+/ufHMx5JOJUmSJCWuKEVrJdBu\nC6+3W//ehvOuKmoolbL0DBiQHa/0jMJ9dtPNjZ/vDa/1d3NjSZIkVWpFKVqDgKEhhIEhhHNDCOeE\nEAYC9wB3rT/mOGBmcYVUGVe15vrNjf8WP588BB473c2NJUmSVGkVumhFUXQD0BNoS1ysBq1/3DOK\nohvXHzYUOLG4QqocSEmBo/4OZz4cb2785Xi4r4ObG0uSJKlSKtI+WlEUPRZF0SFRFG23fh0SRdHI\njd5fEUXRyt87hyqo/f4U30qY1RCWfAn3HQ2fvp50KkmSJKlUFXnD4hBCq41uHWxZnKFUzu3YNB6S\nsduhkLcURnZ1c2NJkiRVKoUuWiGE+iGEccR7Zt0FDAamhxDeDCFsX9wBVU5l1IvHv7fuAUTx5sbP\n9ITVK5JOJkmSJJW4og7DyAT2X3/bYB3ggPWv3fW7n1TlkpYOXe747ebGj5ySdCpJkiSpxBWlaB0P\n9Iqi6JMNL0RR9DHQBzihuIKpAmlzEZz/fLy58fdzkk4jSZIklbiiFK0UYPUWXl9dxPOpMtj9sPh3\nW9vvm//aO3fCurXJZZIkSZJKSFGK0ThgYAihwYYXQgg7A3cAbxZXMFVAdXaHC17If/72f+GRk2Hp\n94lFkiRJkkpCUYrWZUAt4OsQwhchhM+Br9a/1q84w6kCSs/If1ylBnz1NtxzKHw2NrlMkiRJUjEr\nyobFC6IoOhDoDNxJPACjUxRFraIoWlDcAVWB9XgNdmgKyxfDY6fBmH/AmrykU0mSJEnbrMi/qYqi\n6I0oigZFUXRXFEVjQwi7hhAeKM5wquDq7gUXjYU2PePn794FDx4Pv3y97efOy4UBWfHKy93280mS\nJEmFUJzDK7YDLijG86kyqFINOt8KXR+Falnw3XQY2h4+ejbpZJIkSVKROSVQZcO+J8Kl78CuB8Gq\nHHiyO7x4hRscS5IkqVyyaKnsqN0Qur8M7a8CAkx/EO7rAD/OTTqZJEmSVCgWLZUtqVXg6OvhvGcg\noz78+DEMOxJmPAxRlHQ6SZIkqUDSCnpgCOGZPzik9jZmkfI16gC9JsGzl8AX4+CFvvDlBOhyJ1TL\nTDqdJEmS9LsKc0Ur+w/WfODh4g6oSqxmfTjnaThmAIRU+PBpuPdw+G5G0skkSZKk31XgK1pRFP25\nJINIW5SSAoddCbsdCk9dCL98BcOPhY7/Bwf3hhCSTihJkiRtxt9oqXzYtS1c+nY8nXDdani9P4zs\nCrk/J51MkiRJ2oxFS+VH9Tpw5iPQ+TZIrQqfvQ5DD4Wv30k6mSRJkvQbFi2VLyFAm4ug5ziotzcs\nXQQPnQjjb4Z1a5NOJ0mSJAEWLZW29AwYkB2v9Iyin2fHA+DiCdDiHIjWwVu3wEMnQc7C4koqSZIk\nFZlFS+VXegacfDecMgzSa8L8d+CeQ+HT15NOJkmSpErOoqXyr3lXuORt2Kk5rFgCI8+EsQOSTiVJ\nkqRKzKKliqFuI7jwDTioV/x86rBk80iSJKlSs2ip4kirCifcAt1GxRMKNxh3A6xamlwuSZIkVToW\nLVU8+3SCC8fkP598NwxqDbNHQxQll0uSJEmVhkVLFVPmzvmP6+wOy76HZy+GB46DhbMSiyVJkqTK\nwaKliq/neDj6eqhSAxZMgWFHwouXQ+7ibT93Xi4MyIpXXu62n0+SJEkVgkVLFV9aVWh/FVz2PjQ9\nA4hg+ggYdCBMuRfWrkk6oSRJkioYi5Yqj6yd4bT74c+vwg5NYWU2vHot3Hs4fDUx6XSSJEmqQCxa\nqnx2aweXvAWdb4+nE/74ETzUBZ7sDr8uSDqdJEmSKgCLliqnlFRocyH0nQFtLoKQAh89C4PbwFv/\nhdUrk04oSZKkcsyipcqtxnbQ+Ta45G3Y7VBYswLG3whD2sInLzkOXpIkSUVi0ZIAdmwK3V+G04ZD\nrQbw63wYfQ48eir8NC/pdJIkSSpnLFrSBiFA09Oh7/vQ/mpITYcvxsE97eD16+LhGZIkSVIBWLSk\nTaVnwNH/gD5ToEknWLcG3hsMg1rDzMdg3bqkE0qSJKmMs2hJW7PdnnDWKDjnaai7F+T+CM/3huHH\nwLfTk04nSZKkMsyiJf2RxsdAr/eg478hvSZ8Nx3u7wDP94FlPyWdTpIkSWWQRUsqiLR0OLQf9J0O\nzc+KX5v5KNx7WLK5JEmSVCZZtKTCqLUjnDIULnwDdmoBq5bmv/fF+ORySZIkqUwJkfsEbSaEkAlk\nZ2dnk5mZmXQclVXr1sH7w+GVq/Nfa3wsHHcT1GucXC5JkiQVm5ycHLKysgCyoijKKejnvKIlFVVK\nCrQ4e6PnafDZGLj7YHitP6z4NblskiRJSpRFSyouPcfD3sfH4+AnD4FBB8L7D8C6tUknkyRJUimz\naEnFpW4jOHs0nPs01GsCy3+Gl66Eew+Hr97etnPn5cKArHjl5RZPXkmSJJUYi5ZU3PY6BnpNghP+\nC9Vqww8fwkMnwuhzYclXSaeTJElSKbBoSSUhtQocdAn0mwltekJIhU9ehCFtYeyA304rlCRJUoVj\n0ZJKUo3toPOtcOk7sOeRsDYP3rkDBrWCmY/FkwslSZJU4Vi0pNKww35w3nPQbRRstycs+wGe7w33\nd4BvpiSdTpIkScXMoiWVlhBgn07QezJ0/Bek14KFM+GBY+GpCyH726QTSpIkqZhYtKTSllYVDr0c\n+s2AA88HAnz4FAxqDRNugbzlSSeUJEnSNrJoSUmpWR9OGgQXT4CG7WDNCphwMwxuAx88BVGUdEJJ\nkiQVkUVLSlqDFvDnV+D0ByFrV8j5Fp6+EB44Dr6bkXQ6SZIkFYFFSyoLQoADToXLpsFR/x9UqQEL\npsB9R8FzvePhGZIkSSo3QuTtSZsJIWQC2dnZ2WRmZiYdR5VRzsJ4v605o+Pn6RmQlxs/7r8wfi5J\nkqQSl5OTQ1ZWFkBWFEU5Bf2cV7SksiizAZw6DC4cCzu3yi9ZAF9OSCyWJEmSCsaiJZVlu7aJy9aJ\nd+W/9vjZ8FQPWPp9crkkSZL0uyxaUlmXkgJNT89/HlLgw6fj6YRT74N1a5PLJkmSpC2yaEnlzZ9f\ngQYtYVUOvHI13H80LJyVdCpJkiRtxKIllTc7NoOL3oROt0LVTFg4M55O+OpfYWWBf58pSZKkEmTR\nksqjlFRo2zMeB3/AaRCtgylDYUhb+Oi5om92nJcLA7LitfEADkmSJBWKRUsqz2rtCKc/AOc+A3X2\ngKWL4MkL4LEzYMlXSaeTJEmqtCxaUkWw19HQezIc8VdITYfP34C7D4a3b4U1eUmnkyRJqnQsWlJF\nUaUaHNUfer0LexwOa1bCuH/D0MPg63eSTidJklSpWLSkiqZeYzj/BTj1PsjYHhbPgxGd4dlekLs4\n6XSSJEmVgkVLqohCgGZnxsMyWvcAAsweCYNawfSHYN26pBNKkiRVaBYtqSKrXge63AEXvgE7NIWV\nv8KL/eDBE+CHj5JOJ0mSVGFZtKTKYNc2cPEEOPZGqJIBCybDvYfDG9c7xl2SJKkEWLSk8iA9AwZk\nxys9o2jnSE2DdpfBZVNhny6wbg1MGghDDoJ5rxZvXkmSpErOoiVVNlm7QLfH4KzHIashZC+AUd3g\n8XMg57uk00mSJFUIFi2psmpyAvSZDIdeASlpMPcluPeIpFNJkiRVCBYtqTJLz4CO/weXTIRdD4bV\ny/Pf++rt5HJJkiSVcxYtSbDDfvDnV6HzbfmvjeoGI7vC4s+SyyVJklROWbQkxVJSoPlZGz1Pg09f\ng7sPhlf/BsuXFN935eXCgKx4OfVQkiRVQBYtSVvWcxzsfXw8nXDKPXBXS5g8FNauTjqZJElSmWfR\nkrRldfeCs0fDec9C/f3izY5f+yvcfQjMew2iKOmEkiRJZZZFS9Lva9QhHpbR5Q6oUQ9+/gxGdYVH\nToEfPk46nSRJUplk0ZL0x1LToHUP6DcDDr0cUtPhy/Ew9FB48QpY9lPSCSVJksoUi5akgquWBR3/\nBX2mwn5/gmgdTH8QBh0I79wJa1YlnVCSJKlMsGhJKrzt9oAzH4bur8BOzWFVDoz9JwxuAx895++3\nJElSpWfRklR0ux8KPSfAyfdArZ3g1/nw5AXwYCdYODPpdJIkSYmxaEnaNikp0OJs6DsdjvgrpFWH\nb96FYUfBs70gZ1HSCSVJkkqdRUtS8UjPgKP6Q9/3oemZQASzR8a/35rwH8hbnnRCSZKkUmPRklS8\nsnaB0+6Di8bBLm1h9XKYcBMMbg1znoB165JOKEmSVOIsWpJKxi6t4MIxcPoDkNUQcr6DZ3rC8GPg\n22lJp5MkSSpRIXI62GZCCJlAdnZ2NpmZmUnHkcq/1Stg8t0w8XbIW/bb9/ovjG87lCRJKoNycnLI\nysoCyIqiKKegn/OKlqSSV6U6tL8K+s6AlucBIf+9CbfAqqWJRZMkSSoJXtHaAq9oSSVswRQYfmz+\n85o7wNHXQ/Oz4ymGkiRJZYRXtCSVHzsckP+4zu6w7Ad4vg8MOwK+fqf4vicvFwZkxSsvt/jOK0mS\n9AcsWpKS1XM8HHsDVM2E7+fAiM4w+jxY8lXSySRJkorMoiUpWWlVoV1f6DcTWl8IIQU+eQGGtIU3\nroeVBb5CL0mSVGZYtCSVDRn1oMvtcOkk2PMoWJsHkwbGGx5PHwHr1iadUJIkqcAsWpLKlh32g/Oe\nhbOfgLp7Qe5P8OLlcO/h8OVbSaeTJEkqEIuWpLInBNj7OOg9GY6/BarVhh8+hIdPglFnw89fJJ1Q\nkiTpd1m0JJVdqVXg4F7x77faXgIhFea9DEMOgtevgxW/Jp1QkiRpiyxaksq+GttBp/9C7/eg8bGw\nbjW8NxjuaglT74O1a5JOKEmS9BuJF60QQu8QwlchhJUhhOkhhPa/c2z3EEK0hVVto2MGbOH970vn\nr5FUorZvAuc8Cec8DfWawIol8MrVMPRQ+Hxs0ukkSZL+n0SLVgihK3AncCPQEpgIvBpCaPg7H8sB\ndtp4RVG0cpNjPtrkmKbFHF1SkhofA73ehU63QvXt4Ke58Ohp8NgZ8NOnSaeTJElK/IrWX4DhURTd\nH0XRJ1EUXQEsAHr9zmeiKIq+33ht4Zg1mxzzU4mkl5Sc1DRo2xP6zYCD+0BKGnw2Bu4+GF79Kyxf\nknRCSZJUiSVWtEII6UArYMwmb40B2v3OR2uGEOaHEL4NIbwUQmi5hWMahxAWrr8l8fEQwp5/kKVq\nCCFzwwJqFeqPkZSc6nXg+Jug9xRo0gmitTBlaPz7rWn3J51OkiRVUkle0aoHpAI/bPL6D8COW/nM\nXKA7cBJwFrASmBRCaLzRMVOA84HjgJ7rz/VuCKHu72T5O5C90fq2MH+IpEJKz4AB2fFKzyiec9bb\nC84aBec/D/X3h5W/whvX578fRcXzPZIkSQWQ9K2DAJv+20/YwmvxgVE0OYqiR6Momh1F0UTgTOBT\noO9Gx7waRdHTURR9EEXRWKDz+rcu+J0MNwNZG61divanSErcnkfCpROhy51QY6P/vvLIKTD/3aRS\nSZKkSibJorUYWMvmV6/qs/lVri2KomgdMA1o/DvH5AIf/MExq6IoytmwgKUF+X5JZVRKKrT+M1w6\nKf+1b6fCgyfEQzMWzire78vLhQFZ8crLLd5zS5KkcimxohVFUR4wHei4yVsdgQL9Z+cQQgBaAIt+\n55iqwL6/d4ykCqpaZv7jlufFAzM+HwvDjoAnLnBCoSRJKjFJ3zp4O3BRCKFHCGHfEMIdQENgKEAI\n4eEQws0bDg4h/DOEcFwIYc8QQgtgOHHRGrrRMbeGEI4IIewRQjgIeArIBB4qxb9LUllzwn+gz1Ro\neiYQ4OPn4O6D4Pk+8Os3SaeTJEkVTKJFK4qi0cAVwPXALOBwoFMURfPXH9KQeB+sDWoDw4BPiKcT\n7gwcHkXR1I2O2QUYBcwDngHygIM3OqekyqpuIzjtPug1CZp0hmgdzHwUBrWKR8Iv+zHphJIkqYII\nkZO4NrN+xHt2dnY2mZmZf3i8pDIqLxduahA/7r9w8wmHC6bBm/8HX0+Mn1fJgIN7Qbu+UL128X2P\nJEkqt3JycsjKygLIWj/PoUCSvnVQkpKzaxvo/lI8En7nVrA6FybeCgObwcTbHWwhSZKKzKIlSXse\nCRe9CV0fg+33hZXZ8ZWugS1gyjBYk5d0QkmSVM5YtCQJIATYt0v8+61T7oXau0Huj/DqNTC4Fcwa\nCevWJp1SkiSVExYtSdpYSio07waXvQ+db4OaO8ZTCZ/rBXcfAh+/AP62VZIk/QGLliRtSVo6tLkI\n+s2EY/4PqtWGxfPgifNg2JHw+ZsWLkmStFUWLUn6Pek14LAr4Io5cPi18WTCRbPg0VNhRBf4ZkrS\nCSVJUhlk0ZKkgqiWBR2ug8tnw8F9ILUqzH8HHjgWnjg/6XSSJKmMsWhJUmHU3B6Ovwn6zYADz4eQ\nCp+PzX//l68TiyZJksoONyzeAjcsllRgiz+PR8F/8kL8PCUNWveIbzOsuX2y2SRJ0jZzw2JJSkK9\nveCUofnP162BqcPgrhYw4T+walnxfVdeLgzIipebKUuSVKZZtCSpOJ39BOzUAvKWwYSb4sI19T5Y\nuzrpZJIkqRRZtCSpOO1+GPQcD6c/CHX2gNyf4JWrYUhb+OhZR8JLklRJWLQkqbilpMABp0KfqdDp\nVsjYHpZ8CU92h/s6wFdvJ51QkiSVMIuWJJWUtHRo2zPe9PjIv8d7cC2cAQ+dCI+eBt9/kHRCSZJU\nQixaklTSqtaCI/8Gl8+CNj3jyYSfj4Wh7eGZS+CX+UknlCRJxcyiJUmlpWZ96HxrfEvh/qcCEcx5\nHAa3htevg+VLkk4oSZKKiUVLkkpb3UZwxoPx0Iw9Doe1efDeYBjYHCbeBnnLk04oSZK2kUVLkpKy\n84Fw/gtw7tOwQ1NYlQNv/gsGHQjTH4K1a5JOKEmSisiiJUlJCgH2OgYueRtOvQ9qN4Sli+DFfnBP\nO5j7siPhJUkqhyxaklQWpKRAszPhsvfhuJuh+naweB48fjY8cBx8MznphJIkqRAsWpJUlqRVhUN6\nxxMK218NadVhwZS4bD3ZPel0kiSpgELkLSmbCSFkAtnZ2dlkZmYmHUdSZZazCN66BWY8AtHa/Nf7\nzoiHakiSpBKVk5NDVlYWQFYURTkF/ZxXtCSpLMvcCU4cCL0nw94n5L8+9FAYfxOsWpZcNkmStFUW\nLUkqD7bfG04fnv989Qp46z/rJxSOgHVrt/rRQsvLhQFZ8crLLb7zSpJUiVi0JKk8OvU+qLMHLPsB\nXrwchh4Gn411QqEkSWWERUuSyqN9OkOfqfGEwmq14ceP4bHT4JFT4PsPkk4nSVKlZ9GSpPIqLT1/\nQuEhl0FqOnw5Hoa2h+f6QM7CpBNKklRpWbQkqbyrXgeOuzG+wrX/qUAEsx6FQa0cmCFJUkIsWpJU\nUWy3B5zxIFw4FnY9CFYvL7mBGZIk6XdZtCSpotm1DfR4Hc582IEZkiQlxKIlSRVRCLDfnxyYIUlS\nQixaklSROTBDkqREWLQkqTJwYIYkSaXKoiVJlcnWBmbc1TIemLF2TdIJJUmqECxaklQZbTowI/fH\n/IEZX4xLOp0kSeWeRUuSKqstDcz46RMYfW7SySRJKvdC5JjfzYQQMoHs7OxsMjMzk44jSaVjxS/w\n9q0wdRiszYtf2/8UOOo6qNe4ZL4zLxduahA/7r8Q0jNK5nskSSqinJwcsrKyALKiKMop6Oe8oiVJ\nim0YmHHxW/mvffQsDGkLz/aCJV8ml02SpHLGoiVJ+q06u+U/btwRonUweyQMag3PXwa/fpNcNkmS\nygmLliRp6854CC4aB3sdA9FamPkI3HUgvHQlZH+XdDpJksosi5Yk6fft0grOfRp6jIE9j4R1q+H9\nB+CuFvDKtbD0+6QTSpJU5li0JEkF0/AgOP956P4y7HZoPDBj6r0wsDm81h+W/Zh0QkmSygyLliSp\ncHY/LC5b578Qb3q8ZiVMHhIXrjeuh9yfk04oSVLiLFqSpMILAfY8It70+NynYedWsHo5TBoIA5vB\nm/+G5UuSTilJUmIsWpKkogshHpRx0Ztw9hOwU3PIWwYTb42vcE24BVZmJ51SkqRSZ9GSJG27EGDv\n4+I9uLo+BjscAKtyYMLNcGdTePt/sGpp0iklSSo1Fi1JUvEJAfbtApdMhDNGwPb7xFe0xt0AdzaD\nd+6EvNzSz5WXCwOy4pXE90uSKh2LliSp+KWkwP6nQK934bThUHcvWLEExv4zvqXw3cGwekXSKSVJ\nKjEWLUlSyUlJhaanQ+8pcPJQqLM75P4EY66LC9e04UknlCSpRFi0JEklLzUNWpwFl70PJw2CrIaw\n7Ad44x/5x+QuTi6fJEnFzKIlSSo9qVXgwPOh73TocgfU2in/vUGt4Kke8PU7EEXJZZQkqRiEyP8z\n20wIIRPIzs7OJjMzM+k4klRxLV8C/91j89fr7Q2te0DzblC9zrZ/T14u3NQgftx/IaRnbPs5JUmV\nQk5ODllZWQBZURTlFPRzXtGSJCUnrWr+4x6vQavuUCUDFn8Kr/0NbtsHnusN377vVS5JUrli0ZIk\nlQ07NoMTB8JVc6HzbVB/f1izEmY9BvcfDfe2h/cfcD8uSVK5YNGSJJUt1TKhzUXQaxJc+AY0PwtS\nq8L3H8BLV8Jt+8JLf4mfS5JURlm0JEllUwiwa1s4ZWh8leu4m+L9uPKWwvvDYehhcH9HmDXKPbkk\nSWWORUuSVPbV2A4O6ROPh7/gxXgz5JQ0+HYqPHdp/Fuu1/rD4s+STipJEgBpSQeQJKnAQoA9Do/X\n0h9g1qPw/gjI/gYmD4nX7u3jiYX7dIG09KQTS5IqKYuWJKl8qrUDtL8KDr0CvhgXD8r49DX4emK8\nMraHludBqwvix5IklSKLliSpfEtJhcYd4/XrApjxcLyWfQ/v3A7v3AGNjko6pSSpkvE3WpKkiqP2\nrtDhOrjyQzjzEdjzKCCKr3ht8OlrJbcnV14uDMiKV15uyXyHJKlcsGhJkiqe1Cqw30lw/nPQdwYc\n3Cv/vad6wPBj4etJyeWTJFV4Fi1JUsVWtxF0+Ef+87Rq8bTCEZ3g0dPdj0uSVCIsWpKkyqX3e9D6\nwng8/OdvwND28HRPWPJV0skkSRWIRUuSlJz0DBiQHa/0jNL5zpo7QJfboc9UOOA0IIIPnoDBbeCV\na2DZj6WTQ5JUoVm0JEmVU91GcPoDcPFb0OhoWLcapg6DgS1g3I2wMifphJKkcsyiJUmq3Bq0gPOe\ngQtehJ1bwepcePu/MLA5vDcEVq9MOqEkqRyyaEmSBLDH4XDRm/FY+LqNYcUSeL0/DG4NMx+DdWuT\nTihJKkcsWpIkbRBCPBa+92Q4aRDUagDZC+D53nBPO5j7csntwSVJqlAsWpIkbSo1DQ48H/rNgI7/\nhmq14ae58PjZ7sElSSoQi5YkSVtTpToc2g8unw3tr4K06u7BJUkqEIuWJEl/pHptOPp6uHyWe3BJ\nkgrEoiVJUkHV2vEP9uD6KemEkqQywqIlSVJhbW0PrnsOSTqZJKmMsGhJklRUm+3BtTz/vfE3wtLv\nk8smSUqURUuSpG21YQ+uU+/Pf+29IXBnU3ihHyz+vHi/Ly8XBmTFKy+3eM8tSSoWFi1JUsWXngED\nsuOVnlEy3xEC7NMp//kubWBtHsx4KN70ePR58N30kvluSVKZY9GSJKkknP889Hgd9j4BiOCTF+C+\nDjCiC3w+1o2PJamCs2hJklRSGh4MZz8OvSdD87PisfBfT4RHT4N728MHT8HaNUmnlCSVAIuWJEkl\nrf6+cMpQ6DcLDu4NVTLizY6fvhAGHQhT74PVK5JOKUkqRhYtSZJKS+1d4fib4coP4ajroEZd+HU+\nvHI13HEAvPU/WPFL0iklScXAoiVJUmmrsR0ccS1c8SF0uhVqN4Tli2H8DXD7/vBaf8j+NumUkqRt\nYNGSJCkp6TWgbU/oOzMeDb/DAbA6FyYPgYHN4dle8OPcpFNKkorAoiVJUtJS06DZGXDpO3DO07B7\ne1i3BmaPhLsPglFnwTdTkk4pSSoEi5YkSWVFCND4GOj+UrwB8j5dgADzXoEHjoUHjod5r0G0Lumk\nkqQ/kJZ0AEmStAW7tIZuj8Hiz2DSQJj9OHzzXrzqNUk6nSTpD3hFS5KksqxeY/jTYLjiA2jXD9Jr\nweJ5+e/PfBTW5CWXT5K0RRYtSZLKg8yd4Nh/x6Phj/x7/uuvXgt3tYRp98OaVcX7nXm5MCArXnm5\nxXtuSargLFqSJJUn1WtDu775z2vuADnfwstXwcAWMGUYrF6ZXD5JEmDRkiSp+KRnwIDseKVnlM53\n9n4PTvgf1GoASxfCq9fEo+En3wOrV5ROBknSZixakiSVZ2nV4KCL4fJZ0Pk2yNwFln0Pr/0N7mwG\n7w72tj9JSoBFS5KkiiCtKrS5CPrNhC53QlZDyP0RxlwXF6537oRVy5JOKUmVhkVLkqSKJC0dWv8Z\n+s2AkwZDnd1h+WIY+0+4sylMvA1WLU06pSRVeBYtSZIqotQqcOB5cNn7cPI9sN2esGIJvPmvuHC9\n9T9YmZ10SkmqsCxakiRVZKlVoMXZ0GcanDIM6jaGFb/A+BviwjXhFljxa9IpJanCsWhJklQZpKZB\n867QZwqcNhzqNYmvaE24OS5c426E5UuSyeZ+XZIqIIuWJEmVSUoqND0dek+G0x+E+vvBqhx4+7/x\n0Iw3/wW5PyedUpLKPYuWJEmVUUoKHHAqXDoJznwYdjgA8pbGwzLubApvXG/hkqRtYNGSJKkyS0mB\n/f4El0yEbiNhx2awOhcmDYS72yadTpLKLYuWJEnlTXoGDMiOV3pG8ZwzJQX26QyXvA1njYYGLWH1\nivz3R58LHzz129ckSVuVlnQASZJUhoQATY6HvY+DuS/FBQvgi3HxqpoJ+50Ezc+Chu3igiZJ2oz/\ndJQkSZsLARp17CyUfQAAIABJREFUyH9+6BVQu2E8OGPmozCiMwxsDuNugMWfJ5dTksooi5YkSfpj\nR1wL/WZD91eg5Xnxla3sb+Dt/8HgVnDf0TD1vuRGxEtSGeOtg5IkqWBSUmD3Q+PV6X8w7xWY/Th8\n/iZ89368Xvt7fNth827Q+FhIq5p0aklKhEVLkiQVXpXqcMBp8Vr2YzwoY/Yo+H5O/NuuuS9B9Trx\n+826wS6t49sRJamSsGhJkqRtU7M+HNI7Xj98DHMehzlPwNJFMO3+eG3XKB6g0exMqLNb0oklqcT5\nGy1JklR8dtgPOv4LrvwIznsWmnWFKjVgyRcw/gYY2Awe7AQzHoaV2UmnlaQS4xUtSZJU/FJS46mF\njTrAqmXwyYvxrYVfvQ3zJ8XrlWvivbv2O7l0MuXlwk0N4sf9FxbfHmSStAUWLUmSVLKq1oQWZ8Ur\n+zv44Il4iMZPc+HDp+O1wU/zYOcDk8sqScXEWwclSVLpydoZDrsSek+GiyfAQb2gRt389+87Kh4V\nP30ErMxJKKQkbTuLliRJKn0hQIOWcMIt0HdG/uspafGY+Bcvh9uawLO9YP67EEXJZZWkIigTRSuE\n0DuE8FUIYWUIYXoIof3vHNs9hBBtYVUr6jklSVKCUqvkP75sOnT8N9TbG1Yvh9kj4cETYHBreOcO\nWPpDcjklqRASL1ohhK7AncCNQEtgIvBqCKHh73wsB9hp4xVF0cptPKckSUpaze3h0H7QZyr0GAMt\nz4UqGfDz5zB2ANy+L4w6C+a+DGtXJ51WkrYqRAlfig8hTAFmRFHUa6PXPgGei6Lo71s4vjtwZxRF\ntYvrnFv4fCaQnZ2dTWZmZqH+HkmSVEh/NA1w1VL46DmY+QgsmJL/ekb9eMBGy/OgXuNt/x5J2oKc\nnByysrIAsqIoKvCPRxO9ohVCSAdaAWM2eWsM0O53PlozhDA/hPBtCOGlEELLbTlnCKFqCCFzwwJq\nFfZvkSRJJaRqLTjwPLhwTHylq11fyNgecn+ESQPj2wqHHwczH41HyUtSGZD0rYP1gFRg0xuufwB2\n3Mpn5gLdgZOAs4CVwKQQwob/lFWUc/4dyN5ofVvgv0CSJJWe7ZvAsTfAXz6Bro/C3sdDSIEFk+H5\nPvEAjRf6woJpDtCQlKiyso/Wpv8kDFt4LT4wiiYDk//fgSFMAmYAfYF+RTkncDNw+0bPa2HZkiSp\n7EqtAvueGK+cRfHQjJmPwpIvYcbD8dp+n/g3Xs26xb/9kqRSlPQVrcXAWja/0lSfza9IbVEUReuA\nacCGK1qFPmcURauiKMrZsIClBYsvSZISl7kTtL8qHhPf/eW4WKVVjzdEHvP/we37wOhz4fM3k04q\nqRJJtGhFUZQHTAc6bvJWR+DdgpwjhBCAFsCi4jqnJEkqh0KA3Q+DU++Fq+dBlzugwYGwbg188iI8\ncV7+sbmLk8spqVJI+ooWxLfsXRRC6BFC2DeEcAfQEBgKEEJ4OIRw84aDQwj/DCEcF0LYM4TQAhhO\nXLSGFvSckiSpgquWBa17wMXjode7cFAvqF4n//0hbeGVa+DXb4r/u/NyYUBWvPJyi//8ksqFxH+j\nFUXR6BBCXeB64j2xPgQ6RVE0f/0hDYF1G32kNjCM+NbAbGAmcHgURVMLcU5JklRZ7LA/nHALHHEt\n/HeP+LU1K2HqMJg2HJqeAYddAfX3TTanpAol8aIFEEXR3cDdW3nvyE2eXwlcuS3nlCRJlVBa1fzH\nZ42GyXfDV2/BnMfj1aQzHHYl7NomuYySKoyycOugJElS6dqjPVzwAvQcF08uJMC8l2H4MTCiSzw4\nw/HwkrZBmbiiJUmSKrH0DBiQncx379wq3o/rp0/jzY/nPA5fT4zXTs3jK1z7ngQpqcnkk1RueUVL\nkiRp+73h5CFw+ex4cEaVGrBoNjzZHQa3gekPwZpVSaeUVI5YtCRJkjbI2iUenHHFh3DEX6FabVjy\nBbzYDwY2h3cHw6plSaeUVA5YtCRJkjaVUReO6g9XfgjH3gi1doKli2DMdXDH/jD+Jsj9OemUksow\ni5YkSdLWVK0F7S6Lbyk8aRBs1whW/gpv/QfuPABe/Rtkf5t0SkllkEVLkiTpj6RVhQPPh8umwRkj\nYMdmsHo5TLkHBraA5/rEAzUkaT2LliRJUkGlpML+p8Alb8O5z8Du7WHdapj1KAxpC6PPjYdoSKr0\nHO8uSZJUWCHAXkfHa8E0eOeOeB+uT16M1wbRupLLkJcLNzWIH/dfGI/Jl1RmeEVLkiRpW+zaBs4a\nCb2nQPOzIGy059Y97eCt/8KvC5LLJykRIXLX882EEDKB7OzsbDIzM5OOI0mSypOf5sW3Ef5GgEZH\nQctzoUlnqFJt27/HK1pSqcjJySErKwsgK4qinIJ+zitakiRJxSlrl/zHJw2Kf8dFBF+Mg6d6wG1N\n4JVr/C2XVMH5Gy1JkqSScsBp8bTCJV/BrJHxyvkWpg6L1w5N46tczc6EGtslnVZSMfKKliRJUknb\nbg/ocB1cMSeeVrj/qZCaDj98AK/9Nb7K9cQF8NlYWLc26bSSioFXtCRJkkpLSmr+tMLlS+DDp2Hm\nI/FthB8/F69aDaDF2fGq2yjpxJKKyCtakiRJSaixHbTtGe/JdclEOOhSqF4Hli6EibfCoAPhwc4w\na1Q8+EJSuWLRkiRJStpOzeCE/8BV8+CMEbDXMUCA+e/Ac5fCrU3ghb6wYCo4MVoqF7x1UJIkqaxI\nqwr7nxKv7G9h9iiY+Sj88jXMeDhe9faOB2jse1LpZHKMvFQkXtGSJEkqi7J2gcOvgb4zofvL8WbI\nadVh8afwxvUwqFX+sWvzksspaYssWpIkSWVZSgrsfhicMhSu/hROHAi7tIFoo+mEd7WEl6+CBdO8\ntVAqIyxakiRJ5UW1TGjVHS4aCxe/lf/6il9g2v0w/Jh4iMaEW2DJl4nFlGTRkiRJKp/qNc5/3G0k\nNOsKVWrEBWvCzfFVruHHwrTh8Sh5SaXKYRiSJEnl3Z5Hwj6dYdUymPsSzH4cvnoLFkyJ16t/hb2P\ni8vY3sfFQzcklSiLliRJUnFKz4AB2cl8d9Wa0LxbvHIWwYdPwezR8MMHcQGb+xJUy4qnGjbrBg0P\nhhCSySpVcBYtSZKkiihzJ2jXN14/fARzRsOcJ+MNkaePiFft3eKrXM26Qr29kk4sVSj+RkuSJKmi\n22F/6PgvuPJDOP95aH42pNeEX+fD2/+Fwa3gvg4wZRjkLk46rVQheEVLkiSpskhJjX/PteeR0Pk2\nmPdK/HuuL8bBd9Pj9frfYa9j4qtcTU5INq9Ujlm0JEmSKqP0GtD09Hgt+xE+eCq+vXDRLPj0tXhV\nzYyHbEgqNG8dlCRJquxq1odDesMlb0HvKXDYXyBzF1iVA7NH5R834Wb4aV7JZMjLhQFZ8crLLZnv\nkEqRRUuSJEn56u8Dx/wTrvgALngpnmC4wbuDYEhbuPdweO9uWPpDcjmlMs6iJUmSpM2lpMAe7aHz\n7fmvNe4IKWmwaHb8W67b94FHT4M5T3gVStqEv9GSJElSwZzxEKxeAR89Gw/R+O59+HxsvKpkwL4n\nQvOusMcR8eANqRKzaEmSJJVHSW2MnFEP2vaM189frN+fazT88jXMeTxeNXeMh2w06wo7NnVTZFVK\n3jooSZKkoqnbCI7qD/1mQY8x0PpCqF4Hln0P7w2Ge9vDPe3gnTsg+7uk00qlyqIlSZKkbRMCNDwI\nutwOV30K3UbCvidBajr8+DGMHQB37A8jusDMR2FlTtKJpRLnrYOSJEkqPmnp8d5b+3SGFb/Ax8/H\nwzLmT4KvJ8br5augSad4omGjDpBaJenUUrGzaEmSJKlkVK8DrbrH65f58MGT8e+5Fn8KHz0Trxr1\n4IDTYL+Tkk4rFStvHZQkSVLJq7MbHH419JkKF0+Ag3pBxvawfDFMvRdGdM4/dtmPSaWUio1FS5Ik\nSaUnBGjQEk64Bf4yF855CpqeAWnV8o8Z0haevwx++rRkMuTlwoCseLn/l0qIRUuSJEnJSE2LN0E+\n7X64fE7+62vzYOYjMKQNjOwG89+FKEoup1QEFi1JkiQlr2rN/MfnPw/7dAECfPoqPHgC3H8MfPQc\nrFubWESpMByGIUmSpK1LYmPkXdrAnkfC4s/i/bhmjYLv3ocnL4A6e8AhfaDFOZBeo3RzSYXgFS1J\nkiSVTfUaw4kD4coP4fBr4ymGv3wFr1wNdx4A42+G3MVJp5S2yKIlSZKksq1mfehwHVz5EZzwP6i9\nGyz/Gd66Jd4I+aUr4ecvkk4p/YZFS5IkSeVDegYcdDH0mwlnjIAGB8KalfD+AzCoFYw+FxZMTTql\nBFi0JEmSVN6kpML+p0DPcdD9ZWh8HBDBJy/C8I4w/DiY+zKsW5d0UlViDsOQJElS+RQC7H5YvH6c\nC+8NgjlPwILJ8PhkqNsY2l0GzbpBlWp/fL7ilpcLNzWIH/dfGF+RU6XhFS1JkiSVf/X3gT8NgSs+\ngMOuhKpZ8PNn8OLl8eCMt/4Hy5cknVKViEVLkiRJFUetHeGYAfCXj+C4myFrV8j9CcbfEA/OeOVa\n+PWbpFOqErBoSZIkqeKpWgsO6R0Pzjj1ftixKaxeDlPvhXva5R+3dnVyGVWh+RstSZIkJa+kNkZO\nrQLNzoCmp8NXb8Gku+CLN/PfH9gMmnSG/U6CPY9K5rdcqpAsWpIkSar4QoA9j4zXt9Pg/mPi11dm\nw+yR8UqvBXsfF5euvTpCeo3k8qrcs2hJkiSpcqm/X/7jc5+Fz16PR8PnfAcfPhWvtOrQuCPs9ydo\nfCxUy0wur8oli5YkSZIqr4YHwV4d4sEZ302HT56Hj1+AX+fDJy/EKzUdGh0dX+lqcgJUr5N0apUD\nFi1JkiQpJQV2bROvjv+GRbPjkvXxC/GY+E9fjVdKGuxxeHyla58ukFEv6eQqoyxakiRJ0sZCgAYt\n4tXhH/DTXPh4/ZWuHz+CL8bF66UrYbdD80tX5k5JJ1cZYtGSJEmStiYEqL9vvI78Gyz+PP/2wkWz\n4OuJ8XrlGti1bVy69j0RajdMOrkSZtGSJEmSCqreXtD+qnj98nU8ROPjF+DbqbBgSrxe7w8NWsa/\n5yoNeblwU4P4cf+F8ah8Jc6iJUmSJBVFnd2hXd94ZX8Hc1+KS9f8SbBwZrw2mDYcWpwDGXUTi6vS\nlZJ0AEmSJKncy9oZDroE/vwyXP0pdLkjHpqxwRv/gNv2hlFnx2VszarksqpUWLQkSZKk4lSzPrTu\nAWc9nv/ajs1g3RqY9zI8cR7c1gRevgq+fR+iKLmsKjHeOihJkiSVtB6vwa/fwOxRMOcJWLoIpt0f\nr7p7QfNu0Kwb1N416aQqJl7RkiRJkkpD/X2h47/gyo/gvGehWVeoUgN+/hzG3QB3HgAjusDMx2DV\n0qTTaht5RUuSJEmVS3oGDMhO7vtTUqFRh3itWhr/Zmv2qPxR8V9PhFeujsfEN+8GexwRf0blikVL\nkiRJSkrVWtDynHj9+g3MGQ2zH4+vcs0ZHa9aDaDZmdD8LKi/T9KJVUDeOihJkiSVBbUbwuHXwGXv\nw0VvQpuLoFptWLoQJt0Jdx8E9x4Bk4dC7uLSz5eXCwOy4pWXW/rfX854RUuSJEkqS0KAXVrH67ib\n4NPX46tcn70Oi2bFa8x10PjY+NbC3dsnnVhbYNGSJEmSyqq0qrDfSfHKXQwfPgOzR8abIc97JV7V\naucfv25Ncln1GxYtSZIkqTzIqAcHXRyvH+fCnMdh9uj41sINBjaHJp3i1agDpNdILm8lZ9GSJEmS\nypv6+8AxA6DDP+CzMTCqW/z6il9g1v/f3r1H6VWVdxz//pKYgORCEZA7KHIRi4CBqhQFwXgJLlu0\nFSpWoV3owloFvKLIRay2gqgVlpaLplgUWFRRuQgConLpkhAFQggXJZRbwiWSCVgIJLt/nDPm9WVm\nksCZOZN5v5+19po55+z9nudNdvbMk73ffc6pyoR1q2Rrx5mw/VuqRE0jxkRLkiRJWluNGw8vef3K\n44MvgN9eBfMvqnYxvP3iqmQcbPka2HH/KvHa4KXtxdwjTLQkSZKksWLrPWG7GdUmGovmwvxLqqRr\n4c3wv9dV5fLPwMY7VUnXDjNhs92qDTjUKBMtSZIkaTi0+WDkBDbZuSr7fLKe3boU5l8MC66Bh+ZV\n5RcnwdTNq4Rrx5mw9V4wYWI7MY8xJlqSJEnSWLf+VvDqD1TlD4vhzp9WSwrvvAL67ocbzqjKpGnV\njNiO+8PL3gjrTG078rWWiZYkSZLUS164AexyYFWefhLu/nk103X7pfDEQzD3gqqMn1h9/qt/ieGk\nKW1HvlYx0ZIkSZJ61QvWge3fXJUVK+D+2dVnuuZfDI/eBXddUZWLjoTNXjUyMS17Ar6wWfX9px+o\nlmCuhca1HYAkSZKkUWDcONjyL2DG5+Cfb4R/ugH2Ow622KO6/sCclXXPf2+17HDFinZiXQs4oyVJ\nkiTp2TbaHjY6Cl53FCxdCPN+CJd+orrWP9O1wUthj8Ng13fDuuu3G+8o44yWJEmSpKFN2QR2e8/K\n4z0OqzbOWPw7uOxoOGWnannhonntxTjKmGhJkiRJWjMzToCj5sHbvlI9k+vpJ2D2t+Abr4VZb6tm\nv5Y/03aUrXLpoCRJkrQ2a+t5XZMmw+7/ANMPhXuuhV+dDrddBAt+WZWpm8Puh8KrDoHJG418fC1z\nRkuSJEnSc5fANnvBu86GI26B138c1tuoej7XVZ+Hr+wE338/3De77UhHlImWJEmSpGZM2xz2PQaO\nvBUOOB023x2WL4Obz4Mz94PT94HffLd6ftcYZ6IlSZIkqVkTJlUPRD7sSjjsKtjl3TB+Ejzwa7jw\n8GqW64oT4LF724502JhoSZIkSRo+m0+HA75RbZ6x33EwdQv4w6NwzSnwtVfCuQfD734OpbQdaaNM\ntCRJkiQNv/U2rJ7J9ZGb4MBz4CV7Q1kB8y+Cs98Op70afnUGPPV425E2wl0HJUmSJI2c8RPg5W+r\nykPz4YYz4abvwSO3wyUfgyuObzvCRjijJUmSJKkdG+8I+58MR90Gb/0SvOhlsKxjRmvJfe3F9jyl\njLG1kE1IMhVYsmTJEqZOndp2OJIkSVJvWLEC7vwJfO/vquNPP1A9J6xFfX19TJs2DWBaKaVvdds5\noyVJkiRpdBg3rvrs1hhgoiVJkiRJDTPRkiRJkqSGmWhJkiRJUsNMtCRJkiSpYSZakiRJktQwEy1J\nkiRJapiJliRJkiQ1zERLkiRJkhpmoiVJkiRJDTPRkiRJkqSGTWg7AEmSJEn6o4nrwfFL2o7ieXNG\nS5IkSZIaZqIlSZIkSQ0z0ZIkSZKkhploSZIkSVLDTLQkSZIkqWEmWpIkSZLUMBMtSZIkSWqYiZYk\nSZIkNcxES5IkSZIaZqIlSZIkSQ0bFYlWkg8muTvJk0luTPK61Wx3UJKS5MKu87Pq853lf4YnekmS\nJEn6U60nWkkOBL4K/AuwG/BL4NIkW62i3dbAyXX9gfwE2LSjzGwqZkmSJEkaSuuJFnAUcFYp5cxS\nym2llCOAe4HDB2uQZDxwDnAc8LtBqj1VSlnYURY3HrkkSZIkDaDVRCvJRGA6cHnXpcuBPYdoeizw\ncCnlrCHq7JPkoSR3JDkjycZDxDEpydT+AkxZ3fcgSZIkSd3antHaEBgPLOo6vwjYZKAGSf4S+Efg\nsCFe91LgYGBf4KPAHsBVSSYNUv9oYElHuW8145ckSZKkZ5nQdgC10nWcAc6RZArwX8BhpZRHBn2x\nUs7rOJybZDZwD7A/8P0BmnwROKXjeAomW5IkSZKeo7YTrUeA5Tx79mpjnj3LBbAtsA3w4yT958YB\nJHkG2KGU8tvuRqWUB5PcA2w3UBCllKeAp/qPO15bkiRJktZYq0sHSynLgBuBGV2XZgDXDdBkPrAz\nsGtH+RHws/r7ewe6T5IXAVsCDzYSuCRJkiQNoe0ZLaiW7H2nXt53PfB+YCvgmwBJzgbuL6UcXUp5\nEpjb2TjJYwCllLn18WTgeOC/qRKrbYAvUM2e/WD4344kSZKkXtd6olVKOa+ecTqW6nlXc4GZpZR7\n6ipbASvW4CWXU816vRdYnyrZ+hlwYCllaWOBS5IkSdIgUsqz9pzoefUW70uWLFnC1KlT2w5HkiRJ\nUkv6+vqYNm0awLRSSt/qtmt7e3dJkiRJGnNMtCRJkiSpYSZakiRJktQwEy1JkiRJapiJliRJkiQ1\nzERLkiRJkhpmoiVJkiRJDTPRkiRJkqSGmWhJkiRJUsNMtCRJkiSpYSZakiRJktQwEy1JkiRJatiE\ntgMYzfr6+toOQZIkSVKLnmtOkFJKw6Gs/ZJsDtzXdhySJEmSRo0tSin3r25lE60BJAmwGbC07ViA\nKVRJ3xaMjnjUHvuCwH6glewLAvuBVrIvDK8pwANlDZInlw4OoP4DXO1sdThVOR8AS0sprmXsYfYF\ngf1AK9kXBPYDrWRfGHZr/GfqZhiSJEmS1DATLUmSJElqmInW6PcUcEL9Vb3NviCwH2gl+4LAfqCV\n7AujjJthSJIkSVLDnNGSJEmSpIaZaEmSJElSw0y0JEmSJKlhJlqSJEmS1DATrVEiydFJbkiyNMlD\nSS5MskNXnUlJvp7kkSRPJPlRki3ailnNW81+cHWS0lXObStmDY8khye5OUlfXa5P8taO644HPWA1\n+oHjQQ+qf1aUJF/tOOeY0IMG6QuOC6OEidbosTdwGvAaYAYwAbg8yXoddb4KHAAcBOwFTAYuSjJ+\nhGPV8FmdfgBwBrBpR/nASAapEXEf8Clg97pcBfwwySvq644HvWFV/QAcD3pKkj2A9wM3d11yTOgx\nQ/QFcFwYFdzefZRKshHwELB3KeUXSaYBDwN/X0o5r66zGXAvMLOUcll70Wq4dPeD+tzVwG9KKUe0\nGZtGXpLFwMeBC3A86Fn9/aCUcpbjQW9JMhmYA3wQOIb6797fEXrPYH2hvnY1jgujgjNao9e0+uvi\n+ut04AXA5f0VSikPAHOBPUc2NI2g7n7Q7+B6ecitSU5OMmWkA9PISTI+yUHAesD1OB70pAH6QT/H\ng95xGnBxKeWKrvOOCb1nsL7Qz3FhFJjQdgB6tiQBTgGuKaXMrU9vAiwrpfy+q/qi+prGmEH6AcA5\nwN3AQuDPgS8Cu1AtNdQYkmRnql+o1wEeBw4opcxLsiuOBz1jsH5QX3Y86BF1kj2daglpN39H6CGr\n6AvguDBqmGiNTqcCr6RaY70qAVz/OTYN2A9KKWd0HM5NcicwO8mrSilzRjJADbvbgV2B9YF3Av+Z\nZO8h6jsejE0D9oNSyjzHg96QZEvga8CbSilPrklTHBPGlNXpC44Lo4dLB0eZJF8H3g68oZRyX8el\nhcDEJH/W1WRjqv+x0hgyRD8YyBzgaWC7YQ9MI6qUsqyUclcpZXYp5WjgJuAjOB70lCH6wUAcD8am\n6VT/vm9M8kySZ6g2T/pw/f0iHBN6xZB9YZDNTxwXWmKiNUqkcirwDmDfUsrdXVVupPpHMqOjzaZU\nU8LXjVigGlar0Q8G8gqqtfkPDmtwGg0CTMLxoNf194OBOB6MTVcCO1PNbPaX2VRLxPq/d0zoDUP2\nhVLK8gHaOC60xKWDo8dpwLuBvwKWJulfU72klPJ/pZQlSc4CvpzkUarNEU4GbgEG+yCk1j5D9oMk\n2wIHA5cAjwA7AV8Gfg1c20K8GiZJvgBcSrVr2BSqLZv3Ad7ieNA7huoHjge9o5SylGpjiz9K8gTw\naP9neB0TesOq+oLjwuhiojV6HF5/vbrr/KHArPr7I4FngPOBdan+V+OQQf73QmunVfWDZcB+VMuG\nJlP98nUxcIL9YMx5MfAdquefLKF6TspbSik/ra87HvSGQftB/VkNxwP1c0wQ+HvCqOJztCRJkiSp\nYX5GS5IkSZIaZqIlSZIkSQ0z0ZIkSZKkhploSZIkSVLDTLQkSZIkqWEmWpIkSZLUMBMtSZIkSWqY\niZYkSZIkNcxES5LU05IsSHJE23FIksYWEy1JUk9IckiSxwa4tAdw+gjc34ROknrIhLYDkCSpTaWU\nh9uOYU0kmVhKWdZ2HJKkoTmjJUkaUUmuTvLvSb6UZHGShUmOX82205KcnuShJH1JrkqyS8f1XZL8\nLMnS+vqNSXZPsg/wbWBaklKX4+s2fzLTVF/7QJKLkvwhyW1JXpvkZXXsTyS5Psm2HW22TfLDJIuS\nPJ7khiRv7HzPwNbAV/rv33HtnUluTfJUHctHu97zgiTHJJmVZAlwRpKJSU5N8mCSJ+s6R6/RX4Qk\naViZaEmS2vA+4Ang1cAngGOTzBiqQZIAFwObADOB6cAc4MokG9TVzgHuo1oOOB34V+Bp4DrgCKAP\n2LQuJw9xu88CZwO7AvOB7wL/AXwR2L2uc2pH/cnAJcAbgd2Ay4AfJ9mqvv6OOq5jO+5PkunA+cC5\nwM7A8cCJSQ7piufjwNz6PZ0IfBh4O/AuYAfgPcCCId6PJGmEuXRQktSGm0spJ9Tf35nkQ8B+wE+H\naPMGqmRk41LKU/W5jyX5a+BvqD5ntRVwUillfv9r9zeuZ4NKKWXhasT37VLK+XW7fwOuB04spVxW\nn/sa1QwZVC96E3BTR/tjkhxAlQydWkpZnGQ5sLTr/kcBV5ZSTqyP70iyE1ViNauj3lWllD8mhnUC\ndydwTSmlAPesxnuSJI0gZ7QkSW24uev4QWDjVbSZTjVz9Gi9PO/xJI8DLwH6l/GdApyZ5Iokn+pc\n3vc84ltUf72l69w6SaYCJFmvXgo5L8ljdVw7UiV+Q3k5cG3XuWuB7ZKM7zg3u6vOLKrZttvrZZhv\nWuU7kiSNKBMtSVIbnu46Lqz6Z9I4qoRs166yA3ASQCnleOAVVEsM9wXm1TNLzye+MsS5/phPAt4J\nfAZ4XR3XLcDEVdwnHa/Vea7bE50HpZQ5VAnmZ4F1gfOTXLCKe0mSRpBLByVJa4s5VJ/PeqaUsmCw\nSqWUO4DDXq4VAAABrUlEQVQ7qDae+B5wKPADYBkwfrB2z9PrgFmllB8AJJkMbNNVZ6D7zwP26jq3\nJ3BHKWX5UDcspfQB5wHn1UnWT5JsUEpZ/NzegiSpSc5oSZLWFldQfVbqwiRvTrJNkj2TfL7eWXDd\neie+fZJsneQvqTbFuK1uvwCYnGS/JBsmeWGDsd0FvCPJrvUuiN/l2T9jFwCvT7J5kg3rc18G9kvy\n2STbJ3kf8CGG3qiDJEcmOSjJjkm2B/4WWAgM9JwwSVILTLQkSWuFetOHmcAvgG9RzVqdSzVztAhY\nDryIarfAO6h287sUOK5ufx3wTapZoIepdjtsypHA76l2N/wx1a6Dc7rqHFvH+tv6/v1LAN8FHES1\nq+DngGNLKbNWcb/HgU9SfXbrhvp1Z5ZSVjzvdyJJakSqn1uSJEmSpKY4oyVJkiRJDTPRkiSNCkkO\n7ty2vavc2nZ8kiStCZcOSpJGhSRTgBcPcvnpUooP5ZUkrTVMtCRJkiSpYS4dlCRJkqSGmWhJkiRJ\nUsNMtCRJkiSpYSZakiRJktQwEy1JkiRJapiJliRJkiQ1zERLkiRJkhr2/6tuK4u96aX6AAAAAElF\nTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x22e2fdc9390>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cvresult = pd.DataFrame.from_csv('1_nestimators.csv')\n",
    "\n",
    "cvresult = cvresult.iloc[20:]\n",
    "# plot\n",
    "test_means = cvresult['test-mlogloss-mean']\n",
    "test_stds = cvresult['test-mlogloss-std'] \n",
    "        \n",
    "train_means = cvresult['train-mlogloss-mean']\n",
    "train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "x_axis = range(20, cvresult.shape[0]+20)\n",
    "\n",
    "fig = pyplot.figure(figsize=(10, 10), dpi=100)\n",
    "pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "pyplot.xlabel( 'n_estimators' )\n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "#pyplot.savefig( 'n_estimators4_1.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 第二步：找出合适的最大深度和叶子节点最小权重"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': range(3, 10, 2), 'min_child_weight': range(1, 6, 2)}"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_depth = range(3,10,2)\n",
    "min_child_weight = range(1,6,2)\n",
    "param_test2_1 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.69284, std: 0.01439, params: {'max_depth': 3, 'min_child_weight': 1},\n",
       "  mean: -0.69721, std: 0.01344, params: {'max_depth': 3, 'min_child_weight': 3},\n",
       "  mean: -0.69620, std: 0.01388, params: {'max_depth': 3, 'min_child_weight': 5},\n",
       "  mean: -0.70090, std: 0.01425, params: {'max_depth': 5, 'min_child_weight': 1},\n",
       "  mean: -0.69946, std: 0.01734, params: {'max_depth': 5, 'min_child_weight': 3},\n",
       "  mean: -0.69688, std: 0.01799, params: {'max_depth': 5, 'min_child_weight': 5},\n",
       "  mean: -0.69741, std: 0.01793, params: {'max_depth': 7, 'min_child_weight': 1},\n",
       "  mean: -0.69847, std: 0.02087, params: {'max_depth': 7, 'min_child_weight': 3},\n",
       "  mean: -0.69464, std: 0.01696, params: {'max_depth': 7, 'min_child_weight': 5},\n",
       "  mean: -0.70384, std: 0.02131, params: {'max_depth': 9, 'min_child_weight': 1},\n",
       "  mean: -0.70015, std: 0.02420, params: {'max_depth': 9, 'min_child_weight': 3},\n",
       "  mean: -0.69672, std: 0.01314, params: {'max_depth': 9, 'min_child_weight': 5}],\n",
       " {'max_depth': 3, 'min_child_weight': 1},\n",
       " -0.69283887055516247)"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=47,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_1 = GridSearchCV(xgb2_1, param_grid = param_test2_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch2_1.grid_scores_, gsearch2_1.best_params_,     gsearch2_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': [2, 3, 4], 'min_child_weight': [0.5, 1.0, 1.5]}"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_depth = [2,3,4]\n",
    "min_child_weight = [0.5,1.0,1.5]\n",
    "param_test2_1 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.70082, std: 0.01176, params: {'max_depth': 2, 'min_child_weight': 0.5},\n",
       "  mean: -0.69885, std: 0.01269, params: {'max_depth': 2, 'min_child_weight': 1.0},\n",
       "  mean: -0.69960, std: 0.01151, params: {'max_depth': 2, 'min_child_weight': 1.5},\n",
       "  mean: -0.69638, std: 0.01464, params: {'max_depth': 3, 'min_child_weight': 0.5},\n",
       "  mean: -0.69284, std: 0.01439, params: {'max_depth': 3, 'min_child_weight': 1.0},\n",
       "  mean: -0.69594, std: 0.01517, params: {'max_depth': 3, 'min_child_weight': 1.5},\n",
       "  mean: -0.69221, std: 0.01598, params: {'max_depth': 4, 'min_child_weight': 0.5},\n",
       "  mean: -0.69306, std: 0.01675, params: {'max_depth': 4, 'min_child_weight': 1.0},\n",
       "  mean: -0.69277, std: 0.01230, params: {'max_depth': 4, 'min_child_weight': 1.5}],\n",
       " {'max_depth': 4, 'min_child_weight': 0.5},\n",
       " -0.69220676065795128)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_1 = GridSearchCV(xgb2_1, param_grid = param_test2_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch2_1.grid_scores_, gsearch2_1.best_params_,     gsearch2_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.692207 using {'max_depth': 4, 'min_child_weight': 0.5}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Text(0,0.5,'Log Loss')"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAELCAYAAADz6wBxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xd8lfX5//HXlR1ImBmMiCAygoBh\nyBBBhiiOaql+VRRbadXWUddXq/60tVi331bQ4qqzbq2LtqjIcIEgQ/YGUcMKGzJJcq7fH5874SQk\n5JCTk5NxPR+PPEjuc9/nfA6BvPMZ9/URVcUYY4yprohwN8AYY0z9ZkFijDEmKBYkxhhjgmJBYowx\nJigWJMYYY4JiQWKMMSYoFiTGGGOCYkFijDEmKBYkxhhjghIV7gbUhqSkJO3YsWO4m2GMMfXKokWL\ndqlqclXnNYog6dixIwsXLgx3M4wxpl4RkR8COc+GtowxxgTFgsQYY0xQLEiMMcYExYLEGGNMUCxI\njDHGBMWCxBhjTFAsSIwxxgSlUdxHUl0zV++g2Kec1iWJJjH2V2WMMRWxn45H8Y+vNjFv0x5ioiIY\nfEJrRqWnMLJ7Cmktm4S7acYYU2eIqoa7DSHXv39/rc6d7YXFPhZ8v4eZa7KYuXoHm3fnAtAtNZGR\n6SmM6p5Cnw4tiYyQmm6yMcaEnYgsUtX+VZ5nQRK4TTuzmbUmi5mrs1iweQ9FPqVlk2iGd0thVHoK\nQ7sk0zw+ugZabIwx4WdB4qemgsTf/rxCvlq/k1mrs5i9Nou9uYVERQindGxVOgR2QnJCjb6mMcbU\nJgsSP6EIEn/FPmXJT3uZsTqLWauzWLvjIACdkpoysrsbAuvfsRUxUbZIzhhTf1iQ+Al1kJT3055c\nZq91Q2DfbNzNoWIfibFRDOuazMjuKQzvlkzrhNhaa48xxlSHBYmf2g4SfzkFRczZsMvNrazJYufB\nAkSgb4eWrreSnkK31EREbMLeGFO3WJD4CWeQ+PP5lJVbDzBzzQ5mrs5i+Zb9ALRvEc/I7imMTE9h\n8AmtiYuODHNLjTHGgqSMuhIk5e04kM9sr6fy9fpd5BUWEx8dyZATk0on7FObxYW7mcaYRsqCxE9d\nDRJ/+YXFzNu0u3R58ZZ9eQD0bN+Mkd1TOSM9hZ7tmhNh96wYY2qJBYmf+hAk/lSVdTuymblmB7NW\nZ7H4x734FJITYxnZzQ2BnXZiEk1jrTCBMSZ0LEj81LcgKW9PziG+WOd6Kl+s28nB/CJiIiMY1Lk1\no7q7IbDjWlnZFmNMzbIg8VPfg8RfYbGPBZv3MGt1FrPWZLFpVw4AXVMTGNk9lVHpKfQ5rgVRkXbP\nijEmOBYkfhpSkJRXUrZl1posvv3elW1p0SSaEd1cT2VYVyvbYoypHgsSPw05SPwdyC/kq3W7mLlm\nB5+v3cmenENERgindGzJqO6pjExP4YSkpnbPijEmIHUiSERkDDAZiASeV9WHyz3+ODDC+7IJkKKq\nLbzHfgXc4z12v6q+4h3vB7wMxAPTgJu0ijfRWILEX0nZlpneENia7a5sS8fWTUqHwE6xsi3GmKMI\ne5CISCSwDhgNZAILgHGquqqS838P9FHVX4tIK2Ah0B9QYBHQT1X3isi3wE3APFyQPKGqHx+tLY0x\nSMrL3Jtbes/K3I27OVTkIyE2imFdkxjZPZURVrbFGFNOoEESyvWjA4ANqrrJa9BbwAVAhUECjAPu\n9T4/C/hMVfd4134GjBGRz4FmqvqNd/yfwM+BowaJgbSWTbhicEeuGNyR3ENFzNmwm1neHfbTlm9H\nBPoc14JR6amM7J5C9zZWtsUYE5hQBkl74Ce/rzOBgRWdKCLHA52AWUe5tr33kVnBcXMMmsREMbpH\nKqN7pOLzKau2HWDG6h3MWpPFY5+u5bFP19KueZy3eVcqgztb2RZjTOVCGSQV/Tpb2TjapcC/VLW4\nimsDfk4RuQa4BqBDhw5Hb2kjFhEh9GzfnJ7tm3PzGV3JOpBfWrn4/cVbeG3ej8RFR3DaiW4IbGT3\nFNo0t7ItxpjDQhkkmcBxfl+nAVsrOfdS4Ppy1w4vd+3n3vG0QJ5TVZ8DngM3RxJ4sxu3lGZxXHJK\nBy45pQP5hcXM/34Ps1bvYMbqLGaszgIOl20Z1T2FXu2tbIsxjV0oJ9ujcJPto4AtuMn2y1R1Zbnz\nugGfAp1KVl95k+2LgL7eaYtxk+17RGQB8HtgPm6y/UlVnXa0tthke/BUlfVZ2d4qsB0s+sGVbUlK\niGVk92RGdk/ltC5JJFjZFmMajLBPtqtqkYjcgAuJSOBFVV0pIvcBC1V1qnfqOOAt/yW8XmD8BRc+\nAPeVTLwD13J4+e/H2ER7rRARuqYm0jU1kWuHdy5TtuXjFdt5Z2EmMZERDDyhFaO6pzAqPdXKthjT\nSNgNiSZohcU+Fm7e61aBrcli005XtqVLSkLphH3fDla2xZj6Juz3kdQlFiS16/tdOV7Zlh3M33S4\nbMvwrsmMTE/l9C7JNG9iZVuMqessSPxYkIRPZWVb+h/f0tu8K5XOyVa2xZi6yILEjwVJ3eDKtuwr\nvRGypGzL8a2buP3ru6cyoJOVbTGmrrAg8WNBUjdt2ZfnhsBW72COX9mWoV2SGJWeyvBuySRZ2RZj\nAlZQVMyO/QVs3Z/H1n15bNufzy8HH09iXPWGksO+asuYqrRvEc8Vg47nikHHk3uoiLkbdjPT6618\nvMKVbck4roW3eVcq6W2tbItpvIqKfWQdLGDb/jy27ssv8+e2/fls3ZfPruyCI64b2T2F9LahnZO0\nHompc1SVlVsPlN6zsjRzPwBtm8e5IbD0FE7tnGRlW0yD4fMpu3MOeeHgFxT789nm9SyyDhZQ7Cv7\n87ppTCRtW8TTtnkc7ZrH07aF35/e8SYx1e8v2NCWHwuS+i3rYD6fr9nJzDU7+Gr9LnIPFRMXHcGQ\nzkmMTHcbeLVtHh/uZhpTIVVlf15hheFQMvy0fX8+h4p9Za6LiYqgXfM42pYPCO/Pts3jaRYXFdJe\nugWJHwuShqOgqJj5m/Ywa00WM1bvIHNvHgA92jbjjPQURqan0tvKtphalFNQxLb9eWzZ5wKiTFDs\nz2PbvnzyCovLXBMZIbRpFkfb5nG0bRFPu+aHexAlf7ZqGhP2oVwLEj8WJA1T5WVbYhjRzQ2BndYl\n2cq2mGrLLyxmu18gVNSjOJBfVOYaEUhOiC0NiLbN42nXomzPIjkxlsh68MuOBYkfC5LGYW/OIb5Y\nt5OZa7L4fG0WB/OLiI4UBp3QunR5cYfWVrbFOEXFPnYcLPDmJFwwlPYovODYnXPoiOtaNokuDQfX\ne/ALiuZxpDaLazBL2C1I/FiQND6FxT4W/bCXWWuymLl6Bxu9si0npiR4q8BS6Hd8Syvb0kD5fMqu\n7ILS3kNFw01ZB/MpN3dNYmxU6fyDfziUDDe1bR5PfEzjWeRhQeLHgsRs3pXDzHJlW5rHRzO8WzIj\nu6cwvGuKlW2pJ1SVfbmFbCnpRfgvg93ngmLHgXwKi8v+bIuNiqBdi8MB0c6bn/APiureb9FQWZD4\nsSAx/g7kF/L1+l3MXJ3F7LVZpWVb+h3f0qtcnELn5ISwT3Q2VgfzC8usaCo/3LR1fx75hWVXOEVH\nCqnNyq5oKt+jaNkk2r6nx8iCxI8FialMsU9ZmrmPWauzmLkmi9XbDgDQoVWT0ntWBnRqRWxU4xnO\nCKX8wuLSgNhaSY/iYMGRk9epiXGHl776rXQq+TMpIdZW6oWABYkfCxITqIrKtjSNiWRYVzcENqJ7\nipVtqURhsY/t+/OPCAf/u6/3VDB53bppzOFeRAXDTanN4oi2uaywsCDxY0FiquNw2RY3t7LjQAEi\ncHKaV7YlPYUebZs1iuGSYp+y82BB2WWw5ZbD7swuoPyPk2ZxUYcnqv2Ww5b0Lto0j7MKBXWYBYkf\nCxITrJKyLbPWuCGwpT/tA6BNszhv8y5XtqU+ruhRVfbkHCoz3FQ+MHYcyKeo3BKn+OjIow43tW0R\nb/fw1HMWJH4sSExNyzqYz+drdzJrdRZfrd9JzqFiYqMiGHJiEiO95cXtWtSNsi0H8gtdQOw7HBD+\nQbFtfz4FReXKc0RG0KZ5XNmlr+VusGseb5PXDZ0FiR8LEhNK/mVbZq7ZwU97DpdtGeXVAjs5rUVI\nJoPzDhVXGA7+901kl5u8jhBILVeew3+VU7sW8bRuGmOT18aCxJ8FiaktqsqGrGw3r7I6i4U/7Ckt\n2zK8mxsCO61LUkD3Kxwq8h0uz3HEvRLu8325hUdcl5QQ64VC+YBwf6YkxtqNmCYgFiR+LEhMuOzL\n9cq2rHZlWw54ZVsGdnJlW3q2b07WwfwKexS7Kpi8bh4fXWa4qewNdvGkNo+1pcqmxliQ+LEgMXVB\nkVe2ZWa5si0ljra3REmPIpi9JYw5VhYkfixITF20eVcOm3fneJPaod9bwphjZVvtGlPHdUxqSsek\npuFuhjFBsxk3Y4wxQbEgMcYYExQLEmOMMUGxIDHGGBMUCxJjjDFBsSAxxhgTFAsSY4wxQbEgMcYY\nExQLEmOMMUGxIDHGGBMUCxJjjDFBsSAxxhgTFAsSY4wxQQlpkIjIGBFZKyIbROTOSs65WERWichK\nEXnD7/gjIrLC+7jE7/goEVksIktE5GsROTGU78EYY8zRhayMvIhEAlOA0UAmsEBEpqrqKr9zugB3\nAUNUda+IpHjHzwX6AhlALPCFiHysqgeAp4ELVHW1iFwH3ANcGar3YYwx5uhC2SMZAGxQ1U2qegh4\nC7ig3DlXA1NUdS+AqmZ5x3sAX6hqkarmAEuBMd5jCjTzPm8ObA3hezDGGFOFUAZJe+Anv68zvWP+\nugJdRWSOiMwTkZKwWAqcLSJNRCQJGAEc5z12FTBNRDKBK4CHQ/YOjDHGVCmUQVLRnqHl9/WNAroA\nw4FxwPMi0kJVpwPTgLnAm8A3QJF3zS3AOaqaBrwE/K3CFxe5RkQWisjCnTt3BvtejDHGVCKUQZLJ\n4V4EQBpHDkNlAh+paqGqfg+sxQULqvqAqmao6mhcKK0XkWTgZFWd713/NnBqRS+uqs+pan9V7Z+c\nnFxz78oYY0wZoQySBUAXEekkIjHApcDUcud8iBu2whvC6gpsEpFIEWntHe8N9AamA3uB5iLS1bt+\nNLA6hO/BGGNMFUK2aktVi0TkBuBTIBJ4UVVXish9wEJVneo9dqaIrAKKgdtVdbeIxAFfiQjAAWC8\nqhYBiMjVwHsi4sMFy69D9R6MMcZUTVTLT1s0PP3799eFCxeGuxnGGFOviMgiVe1f1Xl2Z7sxxpig\nWJAYY4wJigWJMcaYoFiQGGOMCUrIVm0ZcywKCwvJzMwkPz8/3E0xtSAuLo60tDSio6PD3RRTAyxI\nTJ2QmZlJYmIiHTt2xFv2bRooVWX37t1kZmbSqVOncDfH1AAb2jJ1Qn5+Pq1bt7YQaQREhNatW1vv\nswGxIDF1hoVI42Hf64bFgsQYY0xQqgwSEeksIrHe58NF5EYRaRH6phlTe/bt28dTTz1VrWsnTZpE\nbm5uDbeoZgwfPpzqVnX48MMPWbWqdB+6gJ7rp59+YsSIEaSnp3PSSScxefLkar22qV8C6ZG8BxR7\nW9q+AHQC3jj6JcbULw01SIJRPkgCERUVxV//+ldWr17NvHnzmDJlyjE/h6l/Alm15fMKMI4FJqnq\nkyLyXagbZhqvif9eyaqtB2r0OXu0a8a9Pzup0sfvvPNONm7cSEZGBqNHjyYlJYV33nmHgoICxo4d\ny8SJE8nJyeHiiy8mMzOT4uJi/vjHP7Jjxw62bt3KiBEjSEpKYvbs2RU+f0JCAtdffz0zZsygZcuW\nPPjgg/zhD3/gxx9/ZNKkSZx//vls3ryZK664gpycHAD+/ve/c+qpp/LBBx8wZcoUPvvsM7Zv387p\np5/Ol19+SZs2bY54nby8PCZMmMCqVatIT08nLy+v9LHp06dz7733UlBQQOfOnXnppZdISEigY8eO\nXHLJJaVtf+ONN8jKymLq1Kl88cUX3H///bz33nsAvPvuu1x33XXs27ePF154gaFDh5Z5/bZt29K2\nbVsAEhMTSU9PZ8uWLfTo0eMYvlumvgmkR1IoIuOAXwH/8Y7Z4m/ToDz88MN07tyZJUuWMHr0aNav\nX8+3337LkiVLWLRoEV9++SWffPIJ7dq1Y+nSpaxYsYIxY8Zw44030q5dO2bPnl1piADk5OQwfPhw\nFi1aRGJiIvfccw+fffYZH3zwAX/6058ASElJ4bPPPmPx4sW8/fbb3HjjjQCMHTuWNm3aMGXKFK6+\n+momTpxYYYgAPP300zRp0oRly5Zx9913s2jRIgB27drF/fffz4wZM1i8eDH9+/fnb387vCdcs2bN\n+Pbbb7nhhhu4+eabOfXUUzn//PN57LHHWLJkCZ07dwagqKiIb7/9lkmTJjFx4kQAtm7dyjnnnHNE\nWzZv3sx3333HwIEDq/EdMfVJID2SCcDvgAdU9XsR6QS8FtpmmcbsaD2H2jB9+nSmT59Onz59AMjO\nzmb9+vUMHTqU2267jTvuuIPzzjvviN/GjyYmJoYxY9xO0r169SI2Npbo6Gh69erF5s2bAXdT5g03\n3MCSJUuIjIxk3bp1pdc/+eST9OzZk0GDBjFu3LhKX+fLL78sDaDevXvTu3dvAObNm8eqVasYMmQI\nAIcOHWLw4MGl15U857hx47jlllsqff5f/OIXAPTr16+03e3atWPatGllzsvOzubCCy9k0qRJNGvW\nrMq/H1O/VRkkqroKuBFARFoCiapq+6SbBktVueuuu/jtb397xGOLFi1i2rRp3HXXXZx55pmlvYmq\nREdHly55jYiIIDY2tvTzoiK3i/Tjjz9OamoqS5cuxefzERcXV3r9li1biIiIYMeOHfh8PiIiKh9M\nqGhpraoyevRo3nzzzSqvOdrS3JJ2R0ZGlra7vMLCQi688EIuv/zy0uAxDVsgq7Y+F5FmItIKWAq8\nJCIV7pNuTH2VmJjIwYMHATjrrLN48cUXyc7OBtwP8aysLLZu3UqTJk0YP348t912G4sXLz7i2mDs\n37+ftm3bEhERwauvvkpxcTHghpMmTJjAG2+8QXp6epkhqfKGDRvG66+/DsCKFStYtmwZAIMGDWLO\nnDls2LABgNzc3DI9nrfffrv0z5KeSnXel6rym9/8hvT0dG699dZjutbUX4HMkTRX1QPAL4CXVLUf\ncEZom2VM7WrdujVDhgyhZ8+efPbZZ1x22WUMHjyYXr16cdFFF3Hw4EGWL1/OgAEDyMjI4IEHHuCe\ne+4B4JprruHss89mxIgRQbXhuuuu45VXXmHQoEGsW7eOpk2bAvDggw8ydOhQhg4dyt/+9jeef/55\nVq+ueIfpa6+9luzsbHr37s2jjz7KgAEDAEhOTubll19m3Lhx9O7dm0GDBrFmzZrS6woKChg4cCCT\nJ0/m8ccfB+DSSy/lscceo0+fPmzcuLHSdvvPkcyZM4dXX32VWbNmkZGRQUZGxhHDXqbhqXKHRBFZ\nDpwJvALcraoLRGSZqvaujQbWBNshse5bvXo16enp4W5Go9SxY0cWLlxIUlJSrb6ufc/rvprcIfE+\n3N7qG70QOQFYH2wDjTHGNAyBTLa/C7zr9/Um4MJQNsqY+mrgwIEUFBSUOfbqq6/Sq1evGn2dTz/9\nlDvuuKPMsU6dOvHBBx8c83OVrL4yprqqDBIRSQOeBIYACnwN3KSqmSFumzH1zvz582vldc466yzO\nOuusWnktY6oSyNDWS8BUoB3QHvi3d8wYY4wJKEiSVfUlVS3yPl4GkkPcLmOMMfVEIEGyS0TGi0ik\n9zEe2B3qhhljjKkfAgmSXwMXA9uBbcBFuLIpxhhjTNVBoqo/qur5qpqsqimq+nPczYnGNBgNtYx8\nbe9Hkp+fz4ABAzj55JM56aSTuPfee6v12qZ+qe4OiVb7wDQoDTVIglGd/UhiY2OZNWsWS5cuZcmS\nJXzyySfMmzcvRC00dUUg1X8rYhsum9D5+E7Yvrxmn7NNLzi78lqjth9JzexHIiIkJCQArnhjYWGh\n7c/eCFS3R3L0uirG1DO2H0nN7UdSXFxMRkYGKSkpjB492vYjaQQq7ZGIyEEqDgwB4kPWImOO0nOo\nDbYfSXD7kURGRrJkyRL27dvH2LFjWbFiBT179gzo78nUT5UGiaom1mZDjKkrbD+S4PYjKdGiRQuG\nDx/OJ598YkHSwFV3aKtx+GYKfP4wFAS/14Sp22w/kprZj2Tnzp3s27cPcPM1M2bMoHv37sf0HKb+\nqe5ke+OwYxUseQ2+/Qec/gfoNwGiYsLdKhMC/vuRnH322aX7kYCbKH/ttdfYsGEDt99+OxEREURH\nR/P0008Dh/cjadu27VHnSapy3XXXceGFF/Luu+8yYsSICvcjycjI4JRTTuHcc8+tsAT7tddey4QJ\nE+jduzcZGRkV7kdSUlTy/vvvp2vXrsDh/Uh8Pl9pr+XSSy/l6quv5oknnuBf//pXpe3eunUrV111\nFdOmTWPbtm386le/ori4GJ/Px8UXX8x5551X7b8TUz9UuR9JQxDUfiSZC2HGn2HzV9DieBh5D/S8\nCI4ytGCOne1NET62H4mpTE3uR9K4pfWHX/0bLn8P4prB+1fDs0Nh3XRoBCFsjDFVCaSMfEWrt/YD\nC4H/9fYnadhEoMsZ0HkkrHwfZv0F3vgfOH4InDERjjsl3C00dYTtR2Iao0C22p0IbAXewC39vRRo\nA6wFrlXV4SFuY9BqfKvdokOw+BX44hHI2Qndz4NRf4LkbjX3Go2MDXM0PvY9r/tqcmhrjKo+q6oH\nVfWAqj4HnKOqbwMtq2jEGBFZKyIbROTOSs65WERWichKEXnD7/gjIrLC+7jE77iIyAMisk5EVovI\njQG8h5oVFQMDroYbl8CIu2HTF/DUIPjoethv+30ZYxqXQFZt+UTkYqBk2cZFfo9V2p0RkUhgCjAa\nyAQWiMhUVV3ld04X4C5giKruFZEU7/i5QF8gA4gFvhCRj1X1AHAlcBzQXVV9JdeERWyCW83V/9fw\n1V9hwfOw7F0YeA2cdis0aRW2phljTG0JpEdyOXAFkOV9XAGMF5F44IajXDcA2KCqm1T1EPAWcEG5\nc64GpqjqXgBVzfKO9wC+8DbSygGWAmO8x64F7lNVX7lrwqdpEox5CH6/CHr+Aub+HSZnuHA51PCK\n+RljjL9AyshvUtWfqWqS9/EzVd2gqnmq+vVRLm0P/OT3daZ3zF9XoKuIzBGReSJSEhZLgbNFpImI\nJAEjcL0QgM7AJSKyUEQ+9no1dUOLDjD2Gbh2Dhx/Ksy8D57oAwtegOLCcLfOGGNCosogEZE0EflA\nRLJEZIeIvCciaQE8d0V1FsoPhUUBXYDhwDjgeRFpoarTgWnAXOBN4BugpB5DLJDvTQD9A3ixknZf\n44XNwp07dwbQ3BqUehJc9hZM+ARadoT/3gpTBsKK98Hnq922mIA01DLytb0fSYni4mL69OljNyOG\nU85umPdMrdymEMjQ1kvAVKAdrkfxb+9YVTI53IsASMOt/ip/zkeqWqiq3+NWgnUBUNUHVDVDVUfj\nQmm93zXveZ9/APSu6MVV9TlV7a+q/ZOTw7TF/PGD4defwLi3IDIG/jUB/jECNlb/7mcTGg01SIJR\nnf1ISkyePNlWZIXLvp/g4ztgUk/45A7YsjjkLxnIZHuyqvoHx8sicnMA1y0AuohIJ2ALbtnwZeXO\n+RDXE3nZG8LqCmzyJupbqOpuEemNC4vpfteMxPVETgfWUZeJQLezocuZsOxtmP0gvPpzOGE4nPFn\naNcnvO2rgx759hHW7FlTo8/ZvVV37hhwR6WP234kNbMfCUBmZib//e9/ufvuu49aF8zUsKzVMGcy\nLH/Xfd3rYhhyE6SEvtZZID2SXSIyXkQivY/xwO6qLlLVItxk/KfAauAdVV0pIveJyPneaZ8Cu0Vk\nFTAbuF1VdwPRwFfe8eeA8d7zATwMXCgiy4GHgKsCf7thFBEJGZfBDQvhrAdh2zJ4bji8eyXs3hju\n1jV6th9Jze1HcvPNN/Poo48etUKxqUE/zoc3LnW3IKz6CAZc425NGPt0rYQIBNYj+TXwd+Bx3BzH\nXGBCIE+uqtNwcx3+x/7k97nitu29tdw5+biVWxU95z7g3EBev06KjoPB10OfK2Duk67C8Kqp0O9X\ncPodkFjxD4jG5Gg9h9pg+5FUfz+S//znP6SkpNCvXz8+//zzQP96zLFShfXT4evH4cdvIL4VDL/L\nhUgYbjuoMkhU9UfgfP9j3tDWpFA1qlGIawYj73Y3Nn7xKCx6CZa+BYOuhVNvhPgW4W5ho2X7kVR/\nP5I5c+YwdepUpk2bRn5+PgcOHGD8+PG89tprlT6nOQbFhW7RzpzJkLUSmqXBmEeg7xUQ0zRszapu\n3/PWqk8xAUlIgXP/D25YAN3OcfeePJEBc56Awvxwt67RsP1IamY/koceeojMzEw2b97MW2+9xciR\nIy1EasKhXJj/LDzRFz64BtQHY5+Fm5bAoN+FNUSg+vuRVP4ri6meVifARS/AkBthxkT47I8w/xnX\nXT15HETa1jGhZPuR1Mx+JKaG5e5xFTPmPwO5u+G4QXDOY27xTh2ag6rWfiQi8qOqdghBe0Kixos2\n1obvv3T7oGxZBEndXFHI7ue6VWANkBXwCx/bj6QO2r/FzZ8uehkKc6DrGBhys7uloBYFWrSx0l9z\nKykfD643Eh9E20wgOg2Dq2bC6n+7O+TfvhzSBrglwx2HhLt1xphQ2LnWzX8se8cNX/W6yC3hTT0p\n3C07qkqDRFUTa7MhpgIi0ON8N3ey5HW3f/zL58CJo+GMe6FNze5xYYJn+5GYavlpAcyZBGv+A1Hx\nrhDs4Ouh5fHhbllAbKvd+qQwz024ff03yD8Avf7Hrfxq2THcLQuaDXM0Po3+e64KG2bA15Pgh68h\nrgUM/K1bwtu0docZKxP00Japg6Lj4bSb3T0nX09yE3ArP3C/vQy7HRLCVAqmhqjqUZeemoajMfwC\nW6niIlj1ofs/vGM5NGsPZz0EfX/ptqaohyxI6qP4ljB6ovvt5fOH3aqOJa/D4Bvg1Bsgtv6NSsbF\nxbF7925at25tYdLAqSq7d++q5ii/AAAY6ElEQVQuc59Mo1CYB9+95m5E3vcDJHWFC55yIwtRMeFu\nXVBsaKsh2LXe7SO/6iNokuR6J/0nQFRsuFsWsMLCQjIzM8nPt3tnGoO4uDjS0tKIjo4Od1NCL2+v\n+2Vv3jOQuwvSToHTboGuZ9epJbwVCXRoy4KkIdmyyC0Z/v5LtzfKiLvdbzsRkeFumTGNz4GtMO8p\nWPgSHMp2i2ROu8XtVVRPet0WJH4aTZCAm8DbOMsFyvZlkNrT3YPS5cx684/XmHpt13q3hHfpW6DF\n0PNCt4S3Hq6ytMn2xkoEThwFJ4yAle/DrPvhjYuhw6luXuW4AeFuoTEN05ZFroji6v+4YeV+V7o5\nywawqrIqFiQNVUSEu5kp/XxY/IorDPnCaOh2ruuh1FJ5aWMatJIRgDmT3JByXHMY+r8w8Hf1fhXl\nsbChrcaiIBvmPe263IU5cPJlMOIuaB7IrsnGmDJ8xYeX8G5fBolt3Q2E/a6sl6smK2NzJH4sSPzk\n7HYVhhf8AxBXxn7o/4ZlDwNj6p3CfFj6hqvOvfd7aN3FzX/0vrherZIMlAWJHwuSCuz7CT5/CJa+\nCTEJrurwoOvCXo7amDopfz8seMH16nOyoF1fGHqrGyqu40t4g2FB4seC5CiyVruikGunQUIqnP4H\n6PsriGwE6/uNqcrB7W4J74IX4dBB6DzSLeHtOLRRrIK0IPFjQRKAH+e5JcM/fuP2Rhl5D/QY26B/\n2zKmUrs3wtwnYMkb4CuCHj935YnanhzultUqCxI/FiQBUoV1n8LMiZC1yv2nOePP7rcwYxqDrd+5\nCfRVH0FkDPS5HE79vfvlqhGy+0jMsROBbmOgy2hY/i7MegBeHQudTneB0r5vuFtoTM1ThU2fuyW8\nmz6H2GZu+Grg7yAxNdytqxcsSMyRIiLh5EvhpLGw8EX48jH4xwjXvR/5R0g6MdwtNCZ4vmK3cdzX\nj8O2JW6O8IyJrk5dXPNwt65esSAxlYuKhUHXQsbl8M3fYe7f3X+8vlfA6XdCs7bhbqExx66owK1W\nnPME7Nnohq1+Nhl6XwrRjawicQ2xORITuOws1ztZ+BJERMGg37l9pONbhLtlxlQt/wAsegm+eQqy\nt0PbDDeElf4zK2xaCZts92NBUsP2fA+zH3DzKHEt3Hr6Ade4jbeMqWuys9z9HwtegIL9cMJwFyCd\nTm8US3iDYUHix4IkRLYtcyu8Nsxwu7wNv9OVXom0EVNTB+zZ5DaR+u51KD4EPS5wS3jb9Ql3y+oN\nCxI/FiQh9v1X7h6ULQvdrm8j/+iGC+y3PRMO25Z6S3g/dEOwJ4+DU2+0RSLVYMt/Te3pNBSumgFr\n/uPukn/nCmjf3y0Z7jQ03K0zjYEqbP7KrcDaOAtiEt39H4Oug8Q24W5dg2dBYmqGiOuFdD3bFbWb\n/RC8ch6ceAaMuhfa9g53C01D5PO5X2DmTHL7gTRNcf/e+v/aFoHUIhvaMqFRmAffPgdf/Q3y97kt\nf0fcDa06hbtlpiEoKoBl77htEXavd5tHDbnJzdHZEt4aY3MkfixIwihvn/ttcd4zrmZR/wkw7HZI\nSAl3y0x9VHAQFr0M30yBg9ugTW83gZ5+gS3yCAELEj8WJHXAgW3wxSOw+J8QFee2IB18A8Q1C3fL\nTH2QvRPmP+P20cnf76rvnnaLqwNnizpCxoLEjwVJHbJrA8z6i1tR06Q1DL0NTvlNg9wUyNSAvZtd\nRYXvXnXDWennwZBbIK1fuFvWKFiQ+LEgqYO2LIIZE+H7L6B5Bxjx/9wuc3aHsQHYvsINia54HyTC\n1X4bchMkdQl3yxoVCxI/FiR12MZZ7h6UbUshpYdbcdP1LBuuaIxU4Ye5bgnvhs/czp39rnR7oTdr\nF+7WNUp2H4mpHzqPhE7D3VDXrL/Am5dAh8GuCmuHgeFunakNPh+s+9gFSOYCaJLkNlY75SqIbxnu\n1pkAWJCY8IuIgJ6/cPehLP6nm5R/8Uzodg6M+hOkpIe7hSYUig65em1zJsOutdCiA5zzf9BnvNVt\nq2dCuo+qiIwRkbUiskFE7qzknItFZJWIrBSRN/yOPyIiK7yPSyq47kkRyQ5l+00ti4x2E+83fufK\nrGz+Gp4aDB9eB/t+CnfrTE0pyHYVeJ/IgI+uc9/3C1+A338HA662EKmHQtYjEZFIYAowGsgEFojI\nVFVd5XdOF+AuYIiq7hWRFO/4uUBfIAOIBb4QkY9V9YD3eH/AblttqGKawrDb3N3JX/0Vvv0HLP+X\n+yFz2q3QtHW4W2iqI2c3fPsszH/W3aR6/GluH5ATz7A5sXoulENbA4ANqroJQETeAi4AVvmdczUw\nRVX3Aqhqlne8B/CFqhYBRSKyFBgDvOMF1GPAZcDYELbfhFuTVnDWA27L088fhnlPuaGvU2+Ewde5\nwDF1374f3RLexf+Eojzodq67ifC4AeFumakhoRzaag/4j0dkesf8dQW6isgcEZknImO840uBs0Wk\niYgkASOA47zHbgCmquq2ELbd1CUtjoOfT4Fr57ob0WbfD5MzXE+luDDcrTOV2bES3r/Gfa8WvuDm\nwa6bD+PesBBpYELZI6mor1p+rXEU0AUYDqQBX4lIT1WdLiKnAHOBncA3uJ5JO+B/vPOP/uIi1wDX\nAHTo0KGab8HUKSnp7ofQj/PdkuFpt7lSGSPvgZN+4SbtTfj98I1bgbX+U4hu6nqUg6+D5mnhbpkJ\nkVD+z8vkcC8CXFBsreCcj1S1UFW/B9biggVVfUBVM1R1NC6U1gN9gBOBDSKyGWgiIhsqenFVfU5V\n+6tq/+Tk5Jp8XybcOgyECdPgsnfd8NZ7v4HnTocNM929CKb2+Xyw9mN44Sx4aYxbxjvibrhlBYx5\n0EKkgQtlj2QB0EVEOgFbgEtx8xr+PgTGAS97Q1hdgU3ePEgLVd0tIr2B3sB0b86kdHMBEclWVdut\npjESga5nuona5e+64a7XfuGGvs6YaCU0aktxIax4z20ktXO1q1Jw9qNuCa/NYTUaIQsSVS0SkRuA\nT4FI4EVVXSki9wELVXWq99iZIrIKKAZu98IjDjfMBXAAGO+FiDFlRUTAyZfAST+HhS/Bl4/B8yMh\n/Xx3D4qV1AiNQzmw+FX45u+w/ydXlWDsc24eJDI63K0ztcxKpJiGpeCgWyH0zd/dnih9xru95K3E\nRs3I3eP2mZn/LOTtcVUITrsFupxpS3gbIKu15ceCpBHK3glf/R8seMEVghz4O7fk1EpuVM++n9zC\nhsWvQGGu2wnztJuhw6Bwt8yEkAWJHwuSRmzvZpj9oNtNL66Zu6Fx4G/t7ulAZa1xJUyWv+O+7vU/\n7j6e1B7hbZepFRYkfixIDNuXw8z7YP10SGzrhrsyxtuuepX56Vu3hHftNIhuAn1/6arwtrCl9I2J\nBYkfCxJTavMcmHGvW57auguM+qObmLfxfbd0ev1nLkB+nOuGAQf8FgZcY2VpGikrI29MRToOgd98\nBmv+63oo7/wS2veDM/4MnYaFu3XhUVwEK993S3izVkKzNBjzMPS5AmITwt06Uw9YkJjGR8Rt2dp1\nDCx9Ez5/CF75GXQeBWfcC21PDncLa8ehXPjuNfjmSVcPK7k7/Pxp6HkRRMWEu3WmHrEgMY1XZBT0\nvQJ6XQQLnneVhp8d5n6QjrwbWp0Q7haGRu4et5pt/tOQuxvSBsCYR1ywWpkZUw0WJMZEx8Opv3dD\nOXOfcHtlrPoQ+k2AYbdDYmq4W1gz9m9xFZQXvgSFOe7ej9NucfeC2ByRCYJNthtT3sHtbpfGRa9A\nVJwrOHjqjW75cH20c51bwrvsbVAf9LwQhtwEbXqGu2WmjrNVW34sSEy17N7o9pFf+QHEt3KbbZ1y\nFUTFhrtlgclc6FZgrfmva3PfX8LgG6Dl8eFumaknLEj8WJCYoGz9DmZMhE2zoflxMOL/Qe9L3B3z\ndY2qq4I8ZxJs/griWridJQf8FhKsCrY5NhYkfixITI3YONvtg7JtiStSOOpPboK6LswvFBe5eZ2v\nJ8GO5ZDYzt1A2O9XEJsY7taZesruIzGmpnUeAZ1Oh9Ufwcy/wJuXwnGDYPTE8NWcKsyDJa/DnCdg\n3w/uJssLpkCvi20Jr6k1FiTGHIuICDhpLHQ/D757FT5/BF48yxUxHPWn2qtBlbfPLVme/wzk7HQ3\nVZ71gNsP3ZbwmlpmQWJMdURGQ/9fQ+9L3f0YX0+Gp0+Fky91cyihqkl1YBvMmwILX4ZDB93GXkNu\nho6n1Y0hNtMo2RyJMTUhdw98/TeY/xygbnXX0NsqrFGVW5jL1uytbMnewv5D+4mLjKNJdBPio+Jp\nEuX+jI+KLz0WFREFuzbA3Mmw9C3wFbk96ofcBG171/57NY2GTbb7sSAxtWZ/JoWzH2D7ynfJjEtg\nS7fRbEnpypa8nWzJ3kJmdiZ78vcc01PGIMQXFxGv0CS2GfEJbYmPa3HU8Kn0WHR86eexkbGI9WLM\nUdhkuzEh4lMfO3NdMJSEw5aDW0q/3pG7A19aW3fy7nlE7fqGNjHNad+6OyOOG0H7hPbuI7E9rWJb\nkVecR15RHrmFuYf/3L6EvA3Tyd37PXlRceS2OYm8pBPJQ8ktcuftzN1JXpF3rXesyBf4jtQRElFl\nb6j0WPThz0se83/c/1hpL8o0GvbdNqYcVeXAoQNHBETJ11uzt3LId6jMNcnxyaQlptEvtV9pUKQl\nptH+wC5S5jxJ1Lo50DIbRoxxd5ZXNCHuK4ZVH7l7QLYthYQ2MPh26HdlwHfVFxYXloZKyZ95hYc/\nLwkr/8fLH8spymFn3uGQKvk4FjERMWXDp1xvKJBAqijErBdVN9nQlmmU/OcpMrMzXVj4hUZ2YXaZ\n85vFNDscDiU9Cq9X0a5pO+Ki4ip/MVXYMMPd1LhjObTpBaP+DCeOchPkhfmw9A2Y+yTs2QStT3Ql\nWU6+tM7cRe9TH/lF+UeET2lYVXTMP8CKcskrzCsbYt7XRXpsvahAwueIY9EV97j8r4msizeYhpnN\nkfixIGl8Cn2FbM/efjgk/IKionmKuMi40mAo7VEkpJV+nRhTAzf1+Xyw4l8w6353z0fHoW611YIX\nICcL2vVxRRS7n1c375oPkSN6URUE0NF6VCWBVD7U8ovzj6kdsZGxRw+kas5JxUTE1NtelAWJHwuS\nhsenPnbl7XLBcNAvLLzA2J67HZ/6Ss+PlEjaNG1DWmKaCwi/HkX7hPa0jmtde//Ziw7Bopfhy0fd\nPSAnjIDTbnY3O9bTHzh1UZle1FHCqapjFX1erMUBtyNCIo4tfKpYJOE/FxXqXpQFiR8LkvqnuvMU\n/uFQGhiJ7Ultklr3JoALst1+IFZEsV5RVQp9hRX3jioY4itzXhVzVNXpRVW1SOL6jOtJaZJSrfdq\nq7ZMnZdXlFdmuCnQeYouLbsw/LjhxzZPURfFJthWtvWQiBATGUNMZAwtaFGjz13sKya/OP/IwCnf\nW6pijmp77vbSY1f1uqpG21gRCxITMoW+QrbnbD9ifmLLwcrnKdoltCMtMY2+qX1DM09hTB0WGRFJ\n04imNI1uGu6mHBMLElNt1Z6nSEgr26PwVkPV6jyFMabGWJCYSgUzT9EntU/9mKcwxgTN/lc3cnlF\neYfvp/DrVZR8Xn6eIjEmkbSEtIYzT2GMCZoFSQNX6TyF9/Xu/N1lzi+Zp2if0J4+KX3K3oCX2J5m\nMfV033JjTMhYkNRzVc1T7MjdUWbNu/88xenHnW7zFMaYoFmQ1AP7C/aXCYeSHkXmwUy25WyjoLig\nzPk2T2GMqU32E6UOqGyeoiQ4DhYeLHN+yTzFiS1O5PS008vcgNcuweYpjDG1y4KkFhzrPEVsZGzp\ncFNGcobNUxhj6jQLkhqgqofnKfyWypZ8bvMUxpiGzIIkQJXNU2zJdvdTlJ+nSIpPcj2KlIwjyo+3\nadrG5imMMQ2G/TQ7ikmLJjFn65yjzlN0bt6ZYe2H2TyFMabRsiA5Cp/6SI5PtnkKY4w5ipAGiYiM\nASYDkcDzqvpwBedcDPwZUGCpql7mHX8EONc77S+q+rZ3/HWgP1AIfAv8VlULQ9H+W/vfGoqnNcaY\nBqWCjaNrhohEAlOAs4EewDgR6VHunC7AXcAQVT0JuNk7fi7QF8gABgK3i0hJF+B1oDvQC4gHQl8j\n2RhjTKVCFiTAAGCDqm5S1UPAW8AF5c65GpiiqnsBVDXLO94D+EJVi1Q1B1gKjPHOmaYeXI8kLYTv\nwRhjTBVCGSTtgZ/8vs70jvnrCnQVkTkiMs8bCgMXHGeLSBMRSQJGAMf5Xygi0cAVwCchab0xxpiA\nhHKOpKIbIcrv6xsFdAGG43oWX4lIT1WdLiKnAHOBncA3QFG5a58CvlTVryp8cZFrgGsAOnToUN33\nYIwxpgqh7JFkUrYXkQZsreCcj1S1UFW/B9biggVVfUBVM1R1NC6U1pdcJCL3AslApbPhqvqcqvZX\n1f7Jyck18oaMMcYcKZRBsgDoIiKdRCQGuBSYWu6cD3HDVnhDWF2BTSISKSKtveO9gd7AdO/rq4Cz\ngHGqftvvGWOMCYuQDW2papGI3AB8ilv++6KqrhSR+4CFqjrVe+xMEVkFFAO3q+puEYnDDXMBHADG\nq2rJ0NYzwA/AN97j76vqfaF6H8YYY45O3OKnhq1///66cOHCcDfDGGPqFRFZpKr9qzyvMQSJiOzE\n9WLqkyRgV7gbUcvsPTcO9p7rj+NVtcpJ5kYRJPWRiCwM5DeBhsTec+Ng77nhCeVkuzHGmEbAgsQY\nY0xQLEjqrufC3YAwsPfcONh7bmBsjsQYY0xQrEdijDEmKBYkYSYiY0RkrYhsEJE7KznnYhFZJSIr\nReSN2m5jTavqPYtIBxGZLSLficgyETknHO2sKSLyoohkiciKSh4XEXnC+/tYJiJ9a7uNNS2A93y5\n916XichcETm5tttY06p6z37nnSIixSJyUW21LeRU1T7C9IG7438jcAIQg6t63KPcOV2A74CW3tcp\n4W53Lbzn54Brvc97AJvD3e4g3/Mw3P46Kyp5/BzgY1xNuUHA/HC3uRbe86l+/6bPbgzv2TsnEpgF\nTAMuCneba+rDeiThFcyeLfVVIO9ZgZKNzJpzZLHPekVVvwT2HOWUC4B/qjMPaCEibWundaFR1XtW\n1bkl/6aBeTSAfYUC+D4D/B54D6jv/4/LsCAJr2D2bKmvAnnPfwbGi0gm7je339dO08ImkL+Thuw3\nuB5ZgyYi7YGxuHqBDYoFSXgd654t44DnRaRFiNsVSoG853HAy6qahhv2eVVEGvK/1UD+ThokERmB\nC5I7wt2WWjAJuENVi8PdkJoWyo2tTNUC3bNlnqoWAt+LSMmeLQtqp4k1LpD3/BsOb638jVcNOokG\nNhzgJ5C/kwbH2yLieeBsVd0d7vbUgv7AW17V8iTgHBEpUtUPw9us4DXk3/Lqg2rv2VKrraxZgbzn\nH4FRACKSDsThdspsqKYCv/RWbw0C9qvqtnA3KpREpAPwPnCFqq4Ld3tqg6p2UtWOqtoR+BdwXUMI\nEbAeSVhpEHu2hK/VwQnwPf8v8A8RuQU3xHOlekte6iMReRM3NJnkzfvcC0QDqOozuHmgc4ANQC4w\nITwtrTkBvOc/Aa2Bp7zf0Iu0nhc1DOA9N1h2Z7sxxpig2NCWMcaYoFiQGGOMCYoFiTHGmKBYkBhj\njAmKBYkxxpigWJAYY4wJigWJMXWEiGz2bjqtzrVXiki7mnguY46VBYkxDcOVQLuqTjImFCxIjClH\nRDqKyBoReV5EVojI6yJyhleBeb2IDPA+5nqbb80VkW7etbeKyIve572865tU8jqtRWS69xzP4le8\nUUTGi8i3IrJERJ4VkUjveLaI/FVEFovITBFJ9jZI6g+87p0f7z3N773zlotI91D+nZnGzYLEmIqd\nCEwGegPdgcuA04DbgP8HrAGGqWofXLmPB73rJgEnishY4CXgt6qaW8lr3At87T3HVKADlNYXuwQY\noqoZuNI4l3vXNAUWq2pf4AvgXlX9F7AQuFxVM1Q1zzt3l3fe0167jQkJq7VlTMW+V9XlACKyEpip\nqioiy4GOuA23XhGRLrh6YCU1lXwiciWwDHhWVecc5TWGAb/wrvuviJRs9DQK6Acs8OpQxXO48rEP\neNv7/DVc4cPKlDy2qOR1jAkFCxJjKlbg97nP72sf7v/NX4DZqjpWRDoCn/ud3wXIJrA5i4qK3Qnw\niqreVc3rS5S0uRj7v25CyIa2jKme5sAW7/MrSw6KSHPckNgwoLU3f1GZL/GGrETkbKCld3wmcJGI\npHiPtRKR473HIoCS57wM+Nr7/CCQGMT7MabaLEiMqZ5HgYdEZA6uHH6Jx4GnvD02fgM8XBIIFZgI\nDBORxcCZuH1YUNVVwD3AdBFZBnwGlOzhngOcJCKLgJHAfd7xl4Fnyk22G1MrrIy8MfWIiGSrakK4\n22GMP+uRGGOMCYr1SIwJMRGZANxU7vAcVb0+HO0xpqZZkBhjjAmKDW0ZY4wJigWJMcaYoFiQGGOM\nCYoFiTHGmKBYkBhjjAnK/wcY0dIc7BTK9AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x22e2ef746a0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch2_1.best_score_, gsearch2_1.best_params_))\n",
    "test_means = gsearch2_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch2_1.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(max_depth), len(min_child_weight))\n",
    "train_scores = np.array(train_means).reshape(len(max_depth), len(min_child_weight))\n",
    "\n",
    "for i, value in enumerate(max_depth):\n",
    "    pyplot.plot(min_child_weight, -test_scores[i], label= 'test_max_depth:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'max_depth' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "#pyplot.savefig('max_depth_vs_min_child_weght_1.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 第三步：调整max_depth和min_child_weight之后再次调整n_estimators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "xgb2_3 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=699,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=4,\n",
    "        min_child_weight=0.5,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "modelfit(xgb2_3, X_train, y_train, cv_folds = kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.52543004574812946"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "    #Predict training set:\n",
    "train_predprob = xgb2_3.predict_proba(X_train)\n",
    "logloss = log_loss(y_train, train_predprob)\n",
    "logloss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 第四步：调整参数subsample 和 colsample_bytree "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': [0.6, 0.7, 0.8, 0.9],\n",
       " 'subsample': [0.3, 0.4, 0.5, 0.6, 0.7, 0.8]}"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subsample = [i/10.0 for i in range(3,9)]\n",
    "colsample_bytree = [i/10.0 for i in range(6,10)]\n",
    "param_test3_1 = dict(subsample=subsample, colsample_bytree=colsample_bytree)\n",
    "param_test3_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.69991, std: 0.01428, params: {'colsample_bytree': 0.6, 'subsample': 0.3},\n",
       "  mean: -0.68888, std: 0.01808, params: {'colsample_bytree': 0.6, 'subsample': 0.4},\n",
       "  mean: -0.68742, std: 0.01931, params: {'colsample_bytree': 0.6, 'subsample': 0.5},\n",
       "  mean: -0.68958, std: 0.01570, params: {'colsample_bytree': 0.6, 'subsample': 0.6},\n",
       "  mean: -0.68843, std: 0.01903, params: {'colsample_bytree': 0.6, 'subsample': 0.7},\n",
       "  mean: -0.68407, std: 0.01577, params: {'colsample_bytree': 0.6, 'subsample': 0.8},\n",
       "  mean: -0.69295, std: 0.01890, params: {'colsample_bytree': 0.7, 'subsample': 0.3},\n",
       "  mean: -0.69142, std: 0.01802, params: {'colsample_bytree': 0.7, 'subsample': 0.4},\n",
       "  mean: -0.68969, std: 0.01645, params: {'colsample_bytree': 0.7, 'subsample': 0.5},\n",
       "  mean: -0.68748, std: 0.01342, params: {'colsample_bytree': 0.7, 'subsample': 0.6},\n",
       "  mean: -0.68508, std: 0.01800, params: {'colsample_bytree': 0.7, 'subsample': 0.7},\n",
       "  mean: -0.68305, std: 0.01500, params: {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       "  mean: -0.69365, std: 0.02132, params: {'colsample_bytree': 0.8, 'subsample': 0.3},\n",
       "  mean: -0.69456, std: 0.01936, params: {'colsample_bytree': 0.8, 'subsample': 0.4},\n",
       "  mean: -0.68794, std: 0.01707, params: {'colsample_bytree': 0.8, 'subsample': 0.5},\n",
       "  mean: -0.69182, std: 0.01539, params: {'colsample_bytree': 0.8, 'subsample': 0.6},\n",
       "  mean: -0.68731, std: 0.01472, params: {'colsample_bytree': 0.8, 'subsample': 0.7},\n",
       "  mean: -0.68520, std: 0.01733, params: {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       "  mean: -0.69809, std: 0.02122, params: {'colsample_bytree': 0.9, 'subsample': 0.3},\n",
       "  mean: -0.69487, std: 0.02683, params: {'colsample_bytree': 0.9, 'subsample': 0.4},\n",
       "  mean: -0.69271, std: 0.01991, params: {'colsample_bytree': 0.9, 'subsample': 0.5},\n",
       "  mean: -0.68913, std: 0.01569, params: {'colsample_bytree': 0.9, 'subsample': 0.6},\n",
       "  mean: -0.68704, std: 0.01522, params: {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       "  mean: -0.68790, std: 0.01833, params: {'colsample_bytree': 0.9, 'subsample': 0.8}],\n",
       " {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       " -0.6830455869589932)"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb3_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=56,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=6,\n",
    "        min_child_weight=4,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch3_1 = GridSearchCV(xgb3_1, param_grid = param_test3_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch3_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch3_1.grid_scores_, gsearch3_1.best_params_,     gsearch3_1.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 第五步：调整正则参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [1, 1.5, 2], 'reg_lambda': [0.5, 1, 2]}"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg_alpha = [1,1.5, 2]    #default = 0, 测试0.1,1，1.5，2\n",
    "reg_lambda = [0.5, 1, 2]      #default = 1，测试0.1， 0.5， 1，2\n",
    "\n",
    "param_test5_1 = dict(reg_alpha=reg_alpha, reg_lambda=reg_lambda)\n",
    "param_test5_1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.68046, std: 0.01425, params: {'reg_alpha': 1, 'reg_lambda': 0.5},\n",
       "  mean: -0.68062, std: 0.01380, params: {'reg_alpha': 1, 'reg_lambda': 1},\n",
       "  mean: -0.68227, std: 0.01484, params: {'reg_alpha': 1, 'reg_lambda': 2},\n",
       "  mean: -0.68270, std: 0.01391, params: {'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  mean: -0.68343, std: 0.01415, params: {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  mean: -0.68180, std: 0.01521, params: {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  mean: -0.68319, std: 0.01444, params: {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  mean: -0.68449, std: 0.01371, params: {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  mean: -0.68329, std: 0.01496, params: {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " {'reg_alpha': 1, 'reg_lambda': 0.5},\n",
       " -0.68045585892256344)"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb5_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=56,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=4,\n",
    "        min_child_weight=0.5,\n",
    "        gamma=0,\n",
    "        subsample=0.8,\n",
    "        colsample_bytree=0.7,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch5_1 = GridSearchCV(xgb5_1, param_grid = param_test5_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch5_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch5_1.grid_scores_, gsearch5_1.best_params_,     gsearch5_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [0.5, 1, 1.5], 'reg_lambda': [0.3, 0.5, 0.7]}"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg_alpha = [0.5,1, 1.5]    #default = 0, 测试0.1,1，1.5，2\n",
    "reg_lambda = [0.3,0.5, 0.7]      #default = 1，测试0.1， 0.5， 1，2\n",
    "\n",
    "param_test5_1 = dict(reg_alpha=reg_alpha, reg_lambda=reg_lambda)\n",
    "param_test5_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.68238, std: 0.01666, params: {'reg_alpha': 0.5, 'reg_lambda': 0.3},\n",
       "  mean: -0.68196, std: 0.01521, params: {'reg_alpha': 0.5, 'reg_lambda': 0.5},\n",
       "  mean: -0.68164, std: 0.01423, params: {'reg_alpha': 0.5, 'reg_lambda': 0.7},\n",
       "  mean: -0.68137, std: 0.01624, params: {'reg_alpha': 1, 'reg_lambda': 0.3},\n",
       "  mean: -0.68046, std: 0.01425, params: {'reg_alpha': 1, 'reg_lambda': 0.5},\n",
       "  mean: -0.68303, std: 0.01449, params: {'reg_alpha': 1, 'reg_lambda': 0.7},\n",
       "  mean: -0.68287, std: 0.01483, params: {'reg_alpha': 1.5, 'reg_lambda': 0.3},\n",
       "  mean: -0.68270, std: 0.01391, params: {'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  mean: -0.68272, std: 0.01436, params: {'reg_alpha': 1.5, 'reg_lambda': 0.7}],\n",
       " {'reg_alpha': 1, 'reg_lambda': 0.5},\n",
       " -0.68045585892256344)"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb5_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=56,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=4,\n",
    "        min_child_weight=0.5,\n",
    "        gamma=0,\n",
    "        subsample=0.8,\n",
    "        colsample_bytree=0.7,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch5_1 = GridSearchCV(xgb5_1, param_grid = param_test5_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch5_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch5_1.grid_scores_, gsearch5_1.best_params_,     gsearch5_1.best_score_"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
